Java JavaTokenizer.processToken Examples

Example #1

File: JavaTokenizer.java Project: cherifyass/PMD_JavaTesting_tp3

  public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    StringBuilder stringBuilder = sourceCode.getCodeBuffer();

    // Note that Java version is irrelevant for tokenizing
    LanguageVersionHandler languageVersionHandler =
        LanguageVersion.JAVA_14.getLanguageVersionHandler();
    String fileName = sourceCode.getFileName();
    TokenManager tokenMgr =
        languageVersionHandler
            .getParser(languageVersionHandler.getDefaultParserOptions())
            .getTokenManager(fileName, new StringReader(stringBuilder.toString()));
    Token currentToken = (Token) tokenMgr.getNextToken();

    TokenDiscarder discarder = new TokenDiscarder(ignoreAnnotations);

    while (currentToken.image.length() > 0) {
      discarder.updateState(currentToken);

      if (discarder.isDiscarding()) {
        currentToken = (Token) tokenMgr.getNextToken();
        continue;
      }

      processToken(tokenEntries, fileName, currentToken);
      currentToken = (Token) tokenMgr.getNextToken();
    }
    tokenEntries.add(TokenEntry.getEOF());
  }