public double isBashFile(File file, String data, Project project) {
    ParserDefinition definition =
        LanguageParserDefinitions.INSTANCE.forLanguage(BashFileType.BASH_LANGUAGE);

    Lexer lexer = definition.createLexer(project);
    lexer.start(data);

    int tokenCount = 0;
    Set<IElementType> tokenSet = Sets.newHashSet();
    Set<Integer> modeSet = Sets.newHashSet();
    while (lexer.getTokenType() != BashTokenTypes.BAD_CHARACTER && lexer.getTokenType() != null) {
      tokenSet.add(lexer.getTokenType());
      modeSet.add(lexer.getState());

      lexer.advance();
      tokenCount++;
    }

    double score = 0;
    if (lexer.getTokenType() == BashTokenTypes.BAD_CHARACTER) {
      score -= badCharacterWeight;
    }

    if (tokenCount > 4) {
      score += tokenLimitWeight;
    }

    score += Math.min(0.45, (double) tokenSet.size() * tokenWeight);
    score += Math.min(0.45, (double) modeSet.size() * modeWeight);

    return score;
  }
  @Nullable
  public static DataIndexer<TodoIndexEntry, Integer, FileContent> getTodoIndexer(
      FileType fileType, final VirtualFile virtualFile) {
    final DataIndexer<TodoIndexEntry, Integer, FileContent> extIndexer;
    if (fileType instanceof SubstitutedFileType
        && !((SubstitutedFileType) fileType).isSameFileType()) {
      SubstitutedFileType sft = (SubstitutedFileType) fileType;
      extIndexer =
          new CompositeTodoIndexer(
              getTodoIndexer(sft.getOriginalFileType(), virtualFile),
              getTodoIndexer(sft.getFileType(), virtualFile));
    } else {
      extIndexer = TodoIndexers.INSTANCE.forFileType(fileType);
    }
    if (extIndexer != null) {
      return extIndexer;
    }

    if (fileType instanceof LanguageFileType) {
      final Language lang = ((LanguageFileType) fileType).getLanguage();
      final ParserDefinition parserDef = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
      final TokenSet commentTokens = parserDef != null ? parserDef.getCommentTokens() : null;
      if (commentTokens != null) {
        return new TokenSetTodoIndexer(commentTokens, virtualFile);
      }
    }

    if (fileType instanceof CustomSyntaxTableFileType) {
      return new TokenSetTodoIndexer(ABSTRACT_FILE_COMMENT_TOKENS, virtualFile);
    }

    return null;
  }
Example #3
0
 /**
  * Tries to calculate given line's indent column assuming that there might be a comment at the
  * given indent offset (see {@link #getCommentPrefix(IElementType)}).
  *
  * @param line target line
  * @param indentOffset start indent offset to use for the given line
  * @param lineEndOffset given line's end offset
  * @param fallbackColumn column to return if it's not possible to apply comment-specific indent
  *     calculation rules
  * @return given line's indent column to use
  */
 private int calcIndent(int line, int indentOffset, int lineEndOffset, int fallbackColumn) {
   final HighlighterIterator it = myEditor.getHighlighter().createIterator(indentOffset);
   IElementType tokenType = it.getTokenType();
   Language language = tokenType.getLanguage();
   TokenSet comments = myComments.get(language);
   if (comments == null) {
     ParserDefinition definition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
     if (definition != null) {
       comments = definition.getCommentTokens();
     }
     if (comments == null) {
       return fallbackColumn;
     } else {
       myComments.put(language, comments);
     }
   }
   if (comments.contains(tokenType) && indentOffset == it.getStart()) {
     String prefix = COMMENT_PREFIXES.get(tokenType);
     if (prefix == null) {
       prefix = getCommentPrefix(tokenType);
     }
     if (!NO_COMMENT_INFO_MARKER.equals(prefix)) {
       final int indentInsideCommentOffset =
           CharArrayUtil.shiftForward(
               myChars, indentOffset + prefix.length(), lineEndOffset, " \t");
       if (indentInsideCommentOffset < lineEndOffset) {
         int indent = myEditor.calcColumnNumber(indentInsideCommentOffset, line);
         indentAfterUncomment.put(line, indent - prefix.length());
         return indent;
       }
     }
   }
   return fallbackColumn;
 }
  @Override
  @Nullable
  protected PsiFile createFile(@NotNull final Language lang) {
    if (lang == getTemplateDataLanguage()) {

      final PsiFileImpl file =
          (PsiFileImpl)
              LanguageParserDefinitions.INSTANCE.forLanguage(StdLanguages.HTML).createFile(this);
      file.setContentElementType(EmbeddedPerlTokens.HTML_TEMPLATE_DATA);
      return file;
    }

    if (lang == getBaseLanguage()) {
      return LanguageParserDefinitions.INSTANCE.forLanguage(lang).createFile(this);
    }
    return null;
  }
 @Nullable
 protected PsiFile createFile(Language lang) {
   if (lang != getBaseLanguage()) return null;
   final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
   if (parserDefinition != null) {
     return parserDefinition.createFile(this);
   }
   return null;
 }
  public static boolean isInStringLiteral(PsiElement element) {
    ParserDefinition definition =
        LanguageParserDefinitions.INSTANCE.forLanguage(
            PsiUtilBase.findLanguageFromElement(element));
    if (definition == null) {
      return false;
    }

    return isStringLiteral(element, definition)
        || isStringLiteral(element.getParent(), definition)
        || isStringLiteralWithError(element, definition)
        || isStringLiteralWithError(element.getParent(), definition);
  }
 private static void initState(IElementType root, PsiBuilder builder, ErrorState state) {
   PsiFile file = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY);
   state.completionState = file == null ? null : file.getUserData(COMPLETION_STATE_KEY);
   Language language = file == null ? root.getLanguage() : file.getLanguage();
   state.caseSensitive = language.isCaseSensitive();
   ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
   if (parserDefinition != null) {
     state.commentTokens = parserDefinition.getCommentTokens();
     state.whitespaceTokens = parserDefinition.getWhitespaceTokens();
   }
   PairedBraceMatcher matcher = LanguageBraceMatching.INSTANCE.forLanguage(language);
   state.braces = matcher == null ? null : matcher.getPairs();
   if (state.braces != null && state.braces.length == 0) state.braces = null;
 }
  @Override
  public ASTNode parseContents(ASTNode chameleon) {
    final CharTable table = SharedImplUtil.findCharTableByTree(chameleon);
    final FileElement treeElement =
        new DummyHolder(((TreeElement) chameleon).getManager(), null, table).getTreeElement();
    final PsiFile file = (PsiFile) TreeUtil.getFileElement((TreeElement) chameleon).getPsi();
    PsiFile originalFile = file.getOriginalFile();

    final TemplateLanguageFileViewProvider viewProvider =
        (TemplateLanguageFileViewProvider) originalFile.getViewProvider();

    final Language language = getTemplateFileLanguage(viewProvider);
    final CharSequence chars = chameleon.getChars();

    final PsiFile templateFile = createTemplateFile(file, language, chars, viewProvider);

    final TreeElement parsed = ((PsiFileImpl) templateFile).calcTreeElement();
    Lexer langLexer =
        LanguageParserDefinitions.INSTANCE.forLanguage(language).createLexer(file.getProject());
    final Lexer lexer =
        new MergingLexerAdapter(
            new TemplateBlackAndWhiteLexer(
                createBaseLexer(viewProvider),
                langLexer,
                myTemplateElementType,
                myOuterElementType),
            TokenSet.create(myTemplateElementType, myOuterElementType));
    lexer.start(chars);
    insertOuters(parsed, lexer, table);

    if (parsed != null) {
      final TreeElement element = parsed.getFirstChildNode();
      if (element != null) {
        ((CompositeElement) parsed).rawRemoveAllChildren();
        treeElement.rawAddChildren(element);
      }
    }

    treeElement.subtreeChanged();
    TreeElement childNode = treeElement.getFirstChildNode();

    DebugUtil.checkTreeStructure(parsed);
    DebugUtil.checkTreeStructure(treeElement);
    DebugUtil.checkTreeStructure(chameleon);
    DebugUtil.checkTreeStructure(file.getNode());
    DebugUtil.checkTreeStructure(originalFile.getNode());

    return childNode;
  }
Example #9
0
 private static boolean isLiteral(PsiElement element) {
   if (element == null) return false;
   final ASTNode astNode = element.getNode();
   if (astNode == null) {
     return false;
   }
   final IElementType elementType = astNode.getElementType();
   final ParserDefinition parserDefinition =
       LanguageParserDefinitions.INSTANCE.forLanguage(element.getLanguage());
   if (parserDefinition != null) {
     final TokenSet literals = parserDefinition.getStringLiteralElements();
     return literals.contains(elementType);
   }
   return false;
 }
Example #10
0
  public static boolean isCommentToken(
      @NotNull IElementType tokenType, @NotNull LanguageVersion languageVersion) {
    final Language language = tokenType.getLanguage();
    if (language != languageVersion.getLanguage()) {
      return false;
    }
    boolean inComments = false;

    final ParserDefinition parserDefinition =
        LanguageParserDefinitions.INSTANCE.forLanguage(language);

    if (parserDefinition != null) {
      final TokenSet commentTokens = parserDefinition.getCommentTokens(languageVersion);

      if (commentTokens.contains(tokenType)) {
        inComments = true;
      }
    }
    return inComments;
  }
  public static boolean canHaveStub(@NotNull VirtualFile file) {
    final FileType fileType = file.getFileType();
    if (fileType instanceof LanguageFileType) {
      Language l = ((LanguageFileType) fileType).getLanguage();
      ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
      if (parserDefinition == null) return false;

      final IFileElementType elementType = parserDefinition.getFileNodeType();
      return elementType instanceof IStubFileElementType
          && (((IStubFileElementType) elementType).shouldBuildStubFor(file)
              || IndexingStamp.isFileIndexed(
                  file, INDEX_ID, IndexInfrastructure.getIndexCreationStamp(INDEX_ID)));
    }
    if (fileType.isBinary()) {
      final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
      return builder != null && builder.acceptsFile(file);
    }

    return false;
  }
Example #12
0
    private void doVisitElement(PsiElement element) {
      CompiledPattern pattern = myGlobalVisitor.getContext().getPattern();

      if (myGlobalVisitor.getCodeBlockLevel() == 0) {
        initTopLevelElement(element);
        return;
      }

      if (canBePatternVariable(element) && pattern.isRealTypedVar(element)) {
        myGlobalVisitor.handle(element);
        final MatchingHandler handler = pattern.getHandler(element);
        handler.setFilter(
            new NodeFilter() {
              public boolean accepts(PsiElement other) {
                return canBePatternVariableValue(other);
              }
            });

        super.visitElement(element);

        return;
      }

      super.visitElement(element);

      if (myGlobalVisitor.getContext().getSearchHelper().doOptimizing()
          && element instanceof LeafElement) {
        ParserDefinition parserDefinition =
            LanguageParserDefinitions.INSTANCE.forLanguage(element.getLanguage());
        if (parserDefinition != null) {
          String text = element.getText();

          // todo: support variables inside comments
          boolean flag = true;
          if (StringUtil.isJavaIdentifier(text) && flag) {
            myGlobalVisitor.processTokenizedName(
                text, true, GlobalCompilingVisitor.OccurenceKind.CODE);
          }
        }
      }
    }
Example #13
0
        @Override
        public boolean acceptInput(final VirtualFile file) {
          if (!(file.getFileSystem() instanceof LocalFileSystem)) {
            return false; // do not index TODOs in library sources
          }

          final FileType fileType = file.getFileType();
          if (ProjectUtil.isProjectOrWorkspaceFile(file, fileType)) {
            return false;
          }

          if (fileType instanceof LanguageFileType) {
            final Language lang = ((LanguageFileType) fileType).getLanguage();
            final ParserDefinition parserDef = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
            final TokenSet commentTokens = parserDef != null ? parserDef.getCommentTokens() : null;
            return commentTokens != null;
          }

          return IdTableBuilding.isTodoIndexerRegistered(fileType)
              || fileType instanceof AbstractFileType;
        }
 private static int getCumulativeVersion() {
   int version = VERSION;
   for (final FileType fileType : FileTypeManager.getInstance().getRegisteredFileTypes()) {
     if (fileType instanceof LanguageFileType) {
       Language l = ((LanguageFileType) fileType).getLanguage();
       ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
       if (parserDefinition != null) {
         final IFileElementType type = parserDefinition.getFileNodeType();
         if (type instanceof IStubFileElementType) {
           version += ((IStubFileElementType) type).getStubVersion();
         }
       }
     } else if (fileType.isBinary()) {
       final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
       if (builder != null) {
         version += builder.getStubVersion();
       }
     }
   }
   return version;
 }
 @NotNull
 @Override
 public PsiElement createDummyHolder(
     @NotNull final String text,
     @NotNull final IElementType type,
     @Nullable final PsiElement context) {
   final DummyHolder result = DummyHolderFactory.createHolder(myManager, context);
   final FileElement holder = result.getTreeElement();
   final Language language = type.getLanguage();
   final ParserDefinition parserDefinition =
       LanguageParserDefinitions.INSTANCE.forLanguage(language);
   assert parserDefinition != null : "No parser definition for language " + language;
   final Project project = myManager.getProject();
   final Lexer lexer = parserDefinition.createLexer(project);
   final PsiBuilder builder =
       PsiBuilderFactory.getInstance().createBuilder(project, holder, lexer, language, text);
   final ASTNode node = parserDefinition.createParser(project).parse(type, builder);
   holder.rawAddChildren((TreeElement) node);
   final PsiElement psi = node.getPsi();
   assert psi != null : text;
   return psi;
 }
  protected SpacingBuilder createSpacingBuilder(CodeStyleSettings settings) {
    final IFileElementType file =
        LanguageParserDefinitions.INSTANCE
            .forLanguage(PythonLanguage.getInstance())
            .getFileNodeType();
    final PyCodeStyleSettings pySettings = settings.getCustomSettings(PyCodeStyleSettings.class);

    final CommonCodeStyleSettings commonSettings =
        settings.getCommonSettings(PythonLanguage.getInstance());
    return new SpacingBuilder(commonSettings)
        .between(CLASS_DECLARATION, STATEMENT_OR_DECLARATION)
        .blankLines(commonSettings.BLANK_LINES_AROUND_CLASS)
        .between(STATEMENT_OR_DECLARATION, CLASS_DECLARATION)
        .blankLines(commonSettings.BLANK_LINES_AROUND_CLASS)
        .between(FUNCTION_DECLARATION, STATEMENT_OR_DECLARATION)
        .blankLines(commonSettings.BLANK_LINES_AROUND_METHOD)
        .between(STATEMENT_OR_DECLARATION, FUNCTION_DECLARATION)
        .blankLines(commonSettings.BLANK_LINES_AROUND_METHOD)
        .after(FUNCTION_DECLARATION)
        .blankLines(commonSettings.BLANK_LINES_AROUND_METHOD)
        .after(CLASS_DECLARATION)
        .blankLines(commonSettings.BLANK_LINES_AROUND_CLASS)
        // Remove excess blank lines between imports, because ImportOptimizer gets rid of them
        // anyway.
        // Empty lines between import groups are handles in PyBlock#getSpacing
        .between(IMPORT_STATEMENTS, IMPORT_STATEMENTS)
        .spacing(0, Integer.MAX_VALUE, 1, false, 0)
        .between(STATEMENT_OR_DECLARATION, STATEMENT_OR_DECLARATION)
        .spacing(0, Integer.MAX_VALUE, 1, false, 1)
        .between(COLON, STATEMENT_LIST)
        .spacing(1, Integer.MAX_VALUE, 0, true, 0)
        .afterInside(COLON, TokenSet.create(KEY_VALUE_EXPRESSION, LAMBDA_EXPRESSION))
        .spaceIf(pySettings.SPACE_AFTER_PY_COLON)
        .afterInside(GT, ANNOTATION)
        .spaces(1)
        .betweenInside(MINUS, GT, ANNOTATION)
        .none()
        .beforeInside(ANNOTATION, FUNCTION_DECLARATION)
        .spaces(1)
        .beforeInside(ANNOTATION, NAMED_PARAMETER)
        .none()
        .afterInside(COLON, ANNOTATION)
        .spaces(1)
        .afterInside(RARROW, ANNOTATION)
        .spaces(1)
        .between(allButLambda(), PARAMETER_LIST)
        .spaceIf(commonSettings.SPACE_BEFORE_METHOD_PARENTHESES)
        .before(COLON)
        .spaceIf(pySettings.SPACE_BEFORE_PY_COLON)
        .after(COMMA)
        .spaceIf(commonSettings.SPACE_AFTER_COMMA)
        .before(COMMA)
        .spaceIf(commonSettings.SPACE_BEFORE_COMMA)
        .between(FROM_KEYWORD, DOT)
        .spaces(1)
        .between(DOT, IMPORT_KEYWORD)
        .spaces(1)
        .around(DOT)
        .spaces(0)
        .aroundInside(AT, DECORATOR_CALL)
        .none()
        .before(SEMICOLON)
        .spaceIf(commonSettings.SPACE_BEFORE_SEMICOLON)
        .withinPairInside(LPAR, RPAR, ARGUMENT_LIST)
        .spaceIf(commonSettings.SPACE_WITHIN_METHOD_CALL_PARENTHESES)
        .withinPairInside(LPAR, RPAR, PARAMETER_LIST)
        .spaceIf(commonSettings.SPACE_WITHIN_METHOD_PARENTHESES)
        .withinPairInside(LPAR, RPAR, FROM_IMPORT_STATEMENT)
        .spaces(0)
        .withinPairInside(LPAR, RPAR, GENERATOR_EXPRESSION)
        .spaces(0)
        .withinPairInside(LPAR, RPAR, PARENTHESIZED_EXPRESSION)
        .spaces(0)
        .before(LBRACKET)
        .spaceIf(pySettings.SPACE_BEFORE_LBRACKET)
        .afterInside(LBRACE, DICT_LITERAL_EXPRESSION)
        .spaceIf(pySettings.SPACE_WITHIN_BRACES, pySettings.DICT_NEW_LINE_AFTER_LEFT_BRACE)
        .beforeInside(RBRACE, DICT_LITERAL_EXPRESSION)
        .spaceIf(pySettings.SPACE_WITHIN_BRACES, pySettings.DICT_NEW_LINE_BEFORE_RIGHT_BRACE)
        .withinPair(LBRACE, RBRACE)
        .spaceIf(pySettings.SPACE_WITHIN_BRACES)
        .withinPair(LBRACKET, RBRACKET)
        .spaceIf(commonSettings.SPACE_WITHIN_BRACKETS)
        .before(ARGUMENT_LIST)
        .spaceIf(commonSettings.SPACE_BEFORE_METHOD_CALL_PARENTHESES)
        .around(DECORATOR_CALL)
        .spacing(1, Integer.MAX_VALUE, 0, true, 0)
        .after(DECORATOR_LIST)
        .spacing(1, Integer.MAX_VALUE, 0, true, 0)
        .aroundInside(EQ, ASSIGNMENT_STATEMENT)
        .spaceIf(commonSettings.SPACE_AROUND_ASSIGNMENT_OPERATORS)
        .aroundInside(EQ, NAMED_PARAMETER)
        .spaceIf(pySettings.SPACE_AROUND_EQ_IN_NAMED_PARAMETER)
        .aroundInside(EQ, KEYWORD_ARGUMENT_EXPRESSION)
        .spaceIf(pySettings.SPACE_AROUND_EQ_IN_KEYWORD_ARGUMENT)
        .around(AUG_ASSIGN_OPERATIONS)
        .spaceIf(commonSettings.SPACE_AROUND_ASSIGNMENT_OPERATORS)
        .aroundInside(ADDITIVE_OPERATIONS, BINARY_EXPRESSION)
        .spaceIf(commonSettings.SPACE_AROUND_ADDITIVE_OPERATORS)
        .aroundInside(MULTIPLICATIVE_OR_EXP, STAR_PARAMETERS)
        .none()
        .around(MULTIPLICATIVE_OR_EXP)
        .spaceIf(commonSettings.SPACE_AROUND_MULTIPLICATIVE_OPERATORS)
        .around(SHIFT_OPERATIONS)
        .spaceIf(commonSettings.SPACE_AROUND_SHIFT_OPERATORS)
        .around(BITWISE_OPERATIONS)
        .spaceIf(commonSettings.SPACE_AROUND_BITWISE_OPERATORS)
        .around(EQUALITY_OPERATIONS)
        .spaceIf(commonSettings.SPACE_AROUND_EQUALITY_OPERATORS)
        .around(RELATIONAL_OPERATIONS)
        .spaceIf(commonSettings.SPACE_AROUND_RELATIONAL_OPERATORS)
        .around(SINGLE_SPACE_KEYWORDS)
        .spaces(1);
  }
 protected Lexer createBaseLexer(TemplateLanguageFileViewProvider viewProvider) {
   return LanguageParserDefinitions.INSTANCE
       .forLanguage(viewProvider.getBaseLanguage())
       .createLexer(viewProvider.getManager().getProject());
 }
  private static boolean isCommentComplete(
      PsiComment comment, CodeDocumentationAwareCommenter commenter, Editor editor) {
    for (CommentCompleteHandler handler :
        Extensions.getExtensions(CommentCompleteHandler.EP_NAME)) {
      if (handler.isApplicable(comment, commenter)) {
        return handler.isCommentComplete(comment, commenter, editor);
      }
    }

    String commentText = comment.getText();
    final boolean docComment = isDocComment(comment, commenter);
    final String expectedCommentEnd =
        docComment ? commenter.getDocumentationCommentSuffix() : commenter.getBlockCommentSuffix();
    if (!commentText.endsWith(expectedCommentEnd)) return false;

    final PsiFile containingFile = comment.getContainingFile();
    final Language language = containingFile.getLanguage();
    ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
    if (parserDefinition == null) {
      return true;
    }
    Lexer lexer = parserDefinition.createLexer(containingFile.getProject());
    final String commentPrefix =
        docComment ? commenter.getDocumentationCommentPrefix() : commenter.getBlockCommentPrefix();
    lexer.start(
        commentText, commentPrefix == null ? 0 : commentPrefix.length(), commentText.length());
    QuoteHandler fileTypeHandler = TypedHandler.getQuoteHandler(containingFile, editor);
    JavaLikeQuoteHandler javaLikeQuoteHandler =
        fileTypeHandler instanceof JavaLikeQuoteHandler
            ? (JavaLikeQuoteHandler) fileTypeHandler
            : null;

    while (true) {
      IElementType tokenType = lexer.getTokenType();
      if (tokenType == null) {
        return false;
      }

      if (javaLikeQuoteHandler != null
          && javaLikeQuoteHandler.getStringTokenTypes() != null
          && javaLikeQuoteHandler.getStringTokenTypes().contains(tokenType)) {
        String text = commentText.substring(lexer.getTokenStart(), lexer.getTokenEnd());
        int endOffset = comment.getTextRange().getEndOffset();

        if (text.endsWith(expectedCommentEnd)
            && endOffset < containingFile.getTextLength()
            && containingFile.getText().charAt(endOffset) == '\n') {
          return true;
        }
      }
      if (tokenType == commenter.getDocumentationCommentTokenType()
          || tokenType == commenter.getBlockCommentTokenType()) {
        return false;
      }
      if (tokenType == commenter.getLineCommentTokenType()
          && lexer.getTokenText().contains(commentPrefix)) {
        return false;
      }
      if (lexer.getTokenEnd() == commentText.length()) {
        if (tokenType == commenter.getLineCommentTokenType()) {
          String prefix = commenter.getLineCommentPrefix();
          lexer.start(
              commentText,
              lexer.getTokenStart() + (prefix == null ? 0 : prefix.length()),
              commentText.length());
          lexer.advance();
          continue;
        } else if (isInvalidPsi(comment)) {
          return false;
        }
        return true;
      }
      lexer.advance();
    }
  }