private static IRegion getToken(IDocument document, IRegion scanRegion, int tokenId) {

    try {

      final String source = document.get(scanRegion.getOffset(), scanRegion.getLength());

      fgScanner.setSource(source.toCharArray());

      int id = fgScanner.getNextToken();
      while (id != ITerminalSymbols.TokenNameEOF && id != tokenId) id = fgScanner.getNextToken();

      if (id == ITerminalSymbols.TokenNameEOF) return null;

      int tokenOffset = fgScanner.getCurrentTokenStartPosition();
      int tokenLength = fgScanner.getCurrentTokenEndPosition() + 1 - tokenOffset; // inclusive end
      return new Region(tokenOffset + scanRegion.getOffset(), tokenLength);

    } catch (InvalidInputException x) {
      return null;
    } catch (BadLocationException x) {
      return null;
    }
  }
Example #2
0
 /** Lex the input and build the list of toks. */
 private ImmutableList<Tok> buildToks(String text, char... chars) throws FormatterException {
   try {
     kN = 0;
     IScanner scanner = ToolFactory.createScanner(true, true, true, "1.8");
     scanner.setSource(chars);
     List<Tok> toks = new ArrayList<>();
     int charI = 0;
     int columnI = 0;
     while (scanner.getCurrentTokenEndPosition() < chars.length - 1
         && scanner.getNextToken() != ITerminalSymbols.TokenNameEOF) {
       int charI0 = scanner.getCurrentTokenStartPosition();
       // Get string, possibly with Unicode escapes.
       String originalTokText = text.substring(charI0, scanner.getCurrentTokenEndPosition() + 1);
       String tokText = new String(scanner.getCurrentTokenSource()); // Unicode escapes removed.
       char tokText0 = tokText.charAt(0); // The token's first character.
       final boolean isToken; // Is this tok a token?
       final boolean isNumbered; // Is this tok numbered? (tokens and comments)
       boolean extraNewline = false; // Extra newline at end?
       List<String> strings = new ArrayList<>();
       if (Character.isWhitespace(tokText0)) {
         isToken = false;
         isNumbered = false;
         boolean first = true;
         for (String spaces : NEWLINE_SPLITTER.split(originalTokText)) {
           if (!first) {
             strings.add("\n");
           }
           if (!spaces.isEmpty()) {
             strings.add(spaces);
           }
           first = false;
         }
       } else if (tokText.startsWith("'") || tokText.startsWith("\"")) {
         isToken = true;
         isNumbered = true;
         strings.add(originalTokText);
       } else if (tokText.startsWith("//") || tokText.startsWith("/*")) {
         // For compatibility with an earlier lexer, the newline after a // comment is its own tok.
         if (tokText.startsWith("//") && originalTokText.endsWith("\n")) {
           originalTokText = originalTokText.substring(0, originalTokText.length() - 1);
           tokText = tokText.substring(0, tokText.length() - 1);
           extraNewline = true;
         }
         isToken = false;
         isNumbered = true;
         strings.add(originalTokText);
       } else if (Character.isJavaIdentifierStart(tokText0)
           || Character.isDigit(tokText0)
           || tokText0 == '.' && tokText.length() > 1 && Character.isDigit(tokText.charAt(1))) {
         // Identifier, keyword, or numeric literal (a dot may begin a number, as in .2D).
         isToken = true;
         isNumbered = true;
         strings.add(tokText);
       } else {
         // Other tokens ("+" or "++" or ">>" are broken into one-character toks, because ">>"
         // cannot be lexed without syntactic knowledge. This implementation fails if the token
         // contains Unicode escapes.
         isToken = true;
         isNumbered = true;
         for (char c : tokText.toCharArray()) {
           strings.add(String.valueOf(c));
         }
       }
       if (strings.size() == 1) {
         toks.add(
             new Tok(isNumbered ? kN++ : -1, originalTokText, tokText, charI, columnI, isToken));
         for (char c : originalTokText.toCharArray()) {
           if (c == '\n') {
             columnI = 0;
           } else {
             ++columnI;
           }
           ++charI;
         }
       } else {
         if (strings.size() != 1 && !tokText.equals(originalTokText)) {
           throw new FormatterException(
               "Unicode escapes not allowed in whitespace or multi-character operators");
         }
         for (String str : strings) {
           toks.add(new Tok(isNumbered ? kN++ : -1, str, str, charI, columnI, isToken));
           for (char c : str.toCharArray()) {
             if (c == '\n') {
               columnI = 0;
             } else {
               ++columnI;
             }
             ++charI;
           }
         }
       }
       if (extraNewline) {
         toks.add(new Tok(-1, "\n", "\n", charI, columnI, false));
         columnI = 0;
         ++charI;
       }
     }
     toks.add(new Tok(kN++, "", "", charI, columnI, true)); // EOF tok.
     --kN; // Don't count EOF tok.
     computeRanges(toks);
     return ImmutableList.copyOf(toks);
   } catch (InvalidInputException e) {
     throw new FormatterException(e.getMessage());
   }
 }
  /**
   * Finds the key defined by the given match. The assumption is that the key is the only argument
   * and it is a string literal i.e. quoted ("...") or a string constant i.e. 'static final String'
   * defined in the same class.
   *
   * @param keyPositionResult reference parameter: will be filled with the position of the found key
   * @param enclosingElement enclosing java element
   * @return a string denoting the key, {@link #NO_KEY} if no key can be found and <code>null</code>
   *     otherwise
   * @throws CoreException if a problem occurs while accessing the <code>enclosingElement</code>
   */
  private String findKey(Position keyPositionResult, IJavaElement enclosingElement)
      throws CoreException {
    ICompilationUnit unit =
        (ICompilationUnit) enclosingElement.getAncestor(IJavaElement.COMPILATION_UNIT);
    if (unit == null) return null;

    String source = unit.getSource();
    if (source == null) return null;

    IJavaProject javaProject = unit.getJavaProject();
    IScanner scanner = null;
    if (javaProject != null) {
      String complianceLevel = javaProject.getOption(JavaCore.COMPILER_COMPLIANCE, true);
      String sourceLevel = javaProject.getOption(JavaCore.COMPILER_SOURCE, true);
      scanner = ToolFactory.createScanner(false, false, false, sourceLevel, complianceLevel);
    } else {
      scanner = ToolFactory.createScanner(false, false, false, false);
    }
    scanner.setSource(source.toCharArray());
    scanner.resetTo(keyPositionResult.getOffset() + keyPositionResult.getLength(), source.length());

    try {
      if (scanner.getNextToken() != ITerminalSymbols.TokenNameDOT) return null;

      if (scanner.getNextToken() != ITerminalSymbols.TokenNameIdentifier) return null;

      String src = new String(scanner.getCurrentTokenSource());
      int tokenStart = scanner.getCurrentTokenStartPosition();
      int tokenEnd = scanner.getCurrentTokenEndPosition();

      if (scanner.getNextToken() == ITerminalSymbols.TokenNameLPAREN) {
        // Old school
        // next must be key string. Ignore methods which do not take a single String parameter (Bug
        // 295040).
        int nextToken = scanner.getNextToken();
        if (nextToken != ITerminalSymbols.TokenNameStringLiteral
            && nextToken != ITerminalSymbols.TokenNameIdentifier) return null;

        tokenStart = scanner.getCurrentTokenStartPosition();
        tokenEnd = scanner.getCurrentTokenEndPosition();
        int token;
        while ((token = scanner.getNextToken()) == ITerminalSymbols.TokenNameDOT) {
          if ((nextToken = scanner.getNextToken()) != ITerminalSymbols.TokenNameIdentifier) {
            return null;
          }
          tokenStart = scanner.getCurrentTokenStartPosition();
          tokenEnd = scanner.getCurrentTokenEndPosition();
        }
        if (token != ITerminalSymbols.TokenNameRPAREN) return null;

        if (nextToken == ITerminalSymbols.TokenNameStringLiteral) {
          keyPositionResult.setOffset(tokenStart + 1);
          keyPositionResult.setLength(tokenEnd - tokenStart - 1);
          return source.substring(tokenStart + 1, tokenEnd);
        } else if (nextToken == ITerminalSymbols.TokenNameIdentifier) {
          keyPositionResult.setOffset(tokenStart);
          keyPositionResult.setLength(tokenEnd - tokenStart + 1);
          IType parentClass = (IType) enclosingElement.getAncestor(IJavaElement.TYPE);
          IField[] fields = parentClass.getFields();
          String identifier = source.substring(tokenStart, tokenEnd + 1);
          for (int i = 0; i < fields.length; i++) {
            if (fields[i].getElementName().equals(identifier)) {
              if (!Signature.getSignatureSimpleName(fields[i].getTypeSignature())
                  .equals("String")) // $NON-NLS-1$
              return null;
              Object obj = fields[i].getConstant();
              return obj instanceof String
                  ? ((String) obj).substring(1, ((String) obj).length() - 1)
                  : NO_KEY;
            }
          }
        }
        return NO_KEY;
      } else {
        keyPositionResult.setOffset(tokenStart);
        keyPositionResult.setLength(tokenEnd - tokenStart + 1);
        return src;
      }
    } catch (InvalidInputException e) {
      throw new CoreException(JavaUIStatus.createError(IStatus.ERROR, e));
    }
  }