@Override
public boolean consume(CodeReader code, Lexer lexer) {
if (!Character.isJavaIdentifierStart(code.peek())) {
return false;
}
int line = code.getCursor().getLine();
int column = code.getCursor().getColumn();
while (Character.isJavaIdentifierPart(code.peek())) {
tmpBuilder.append((char) code.pop());
}
String word = tmpBuilder.toString();
TokenType keywordType = keywordsMap.get(word);
Token token =
tokenBuilder
.setType(keywordType == null ? GenericTokenType.IDENTIFIER : keywordType)
.setValueAndOriginalValue(word, word)
.setURI(lexer.getURI())
.setLine(line)
.setColumn(column)
.build();
lexer.addToken(token);
tmpBuilder.delete(0, tmpBuilder.length());
return true;
}
@Override
public boolean consume(CodeReader code, Lexer output) {
int line = code.getLinePosition();
int column = code.getColumnPosition();
char ch = code.charAt(0);
if ((ch != '#')) {
return false;
}
String tokenValue = read(code);
output.addToken(
Token.builder()
.setLine(line)
.setColumn(column)
.setURI(output.getURI())
.setValueAndOriginalValue(tokenValue)
.setType(CxxTokenType.PREPROCESSOR)
.build());
return true;
}
public static Lexer create(ObjectiveCConfiguration conf) {
return Lexer.builder()
.withCharset(conf.getCharset())
.withFailIfNoChannelToConsumeOneCharacter(false)
// Comments
.withChannel(commentRegexp("//[^\\n\\r]*+"))
.withChannel(commentRegexp("/\\*[\\s\\S]*?\\*/"))
// All other tokens
.withChannel(regexp(LITERAL, "[^\r\n\\s/]+"))
.withChannel(new BlackHoleChannel("[\\s]"))
.build();
}
@Test
public void hash_followed_by_word() {
List<Token> tokens = lexer.lex("#a");
assertThat(tokens, hasToken("#", CppPunctuator.HASH));
assertThat(tokens, hasToken("a", IDENTIFIER));
}
@Test
public void cpp_operators() {
assertThat(lexer.lex("#"), hasToken("#", CppPunctuator.HASH));
assertThat(lexer.lex("##"), hasToken("##", CppPunctuator.HASHHASH));
}
@Test
public void cpp_identifiers() {
assertThat(lexer.lex("lala"), hasToken("lala", IDENTIFIER));
}
@Test
public void cpp_keywords_indented() {
assertThat(lexer.lex(" #define"), hasToken("#define", CppKeyword.DEFINE));
assertThat(lexer.lex("\t#define"), hasToken("#define", CppKeyword.DEFINE));
}
@Test
public void cpp_keywords_with_whitespaces() {
assertThat(lexer.lex("# define"), hasToken("#define", CppKeyword.DEFINE));
assertThat(lexer.lex("#\tinclude"), hasToken("#include", CppKeyword.INCLUDE));
}