public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder stringBuilder = sourceCode.getCodeBuffer();
// Note that Java version is irrelevant for tokenizing
LanguageVersionHandler languageVersionHandler =
LanguageVersion.JAVA_14.getLanguageVersionHandler();
String fileName = sourceCode.getFileName();
TokenManager tokenMgr =
languageVersionHandler
.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(fileName, new StringReader(stringBuilder.toString()));
Token currentToken = (Token) tokenMgr.getNextToken();
TokenDiscarder discarder = new TokenDiscarder(ignoreAnnotations);
while (currentToken.image.length() > 0) {
discarder.updateState(currentToken);
if (discarder.isDiscarding()) {
currentToken = (Token) tokenMgr.getNextToken();
continue;
}
processToken(tokenEntries, fileName, currentToken);
currentToken = (Token) tokenMgr.getNextToken();
}
tokenEntries.add(TokenEntry.getEOF());
}
private void processToken(Tokens tokenEntries, String fileName, Token currentToken) {
String image = currentToken.image;
if (ignoreLiterals
&& (currentToken.kind == JavaParserConstants.STRING_LITERAL
|| currentToken.kind == JavaParserConstants.CHARACTER_LITERAL
|| currentToken.kind == JavaParserConstants.DECIMAL_LITERAL
|| currentToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) {
image = String.valueOf(currentToken.kind);
}
if (ignoreIdentifiers && currentToken.kind == JavaParserConstants.IDENTIFIER) {
image = String.valueOf(currentToken.kind);
}
tokenEntries.add(new TokenEntry(image, fileName, currentToken.beginLine));
}