public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder stringBuilder = sourceCode.getCodeBuffer(); // Note that Java version is irrelevant for tokenizing LanguageVersionHandler languageVersionHandler = LanguageVersion.JAVA_14.getLanguageVersionHandler(); String fileName = sourceCode.getFileName(); TokenManager tokenMgr = languageVersionHandler .getParser(languageVersionHandler.getDefaultParserOptions()) .getTokenManager(fileName, new StringReader(stringBuilder.toString())); Token currentToken = (Token) tokenMgr.getNextToken(); TokenDiscarder discarder = new TokenDiscarder(ignoreAnnotations); while (currentToken.image.length() > 0) { discarder.updateState(currentToken); if (discarder.isDiscarding()) { currentToken = (Token) tokenMgr.getNextToken(); continue; } processToken(tokenEntries, fileName, currentToken); currentToken = (Token) tokenMgr.getNextToken(); } tokenEntries.add(TokenEntry.getEOF()); }