public double isBashFile(File file, String data, Project project) {
ParserDefinition definition =
LanguageParserDefinitions.INSTANCE.forLanguage(BashFileType.BASH_LANGUAGE);
Lexer lexer = definition.createLexer(project);
lexer.start(data);
int tokenCount = 0;
Set<IElementType> tokenSet = Sets.newHashSet();
Set<Integer> modeSet = Sets.newHashSet();
while (lexer.getTokenType() != BashTokenTypes.BAD_CHARACTER && lexer.getTokenType() != null) {
tokenSet.add(lexer.getTokenType());
modeSet.add(lexer.getState());
lexer.advance();
tokenCount++;
}
double score = 0;
if (lexer.getTokenType() == BashTokenTypes.BAD_CHARACTER) {
score -= badCharacterWeight;
}
if (tokenCount > 4) {
score += tokenLimitWeight;
}
score += Math.min(0.45, (double) tokenSet.size() * tokenWeight);
score += Math.min(0.45, (double) modeSet.size() * modeWeight);
return score;
}
@Nullable
public static DataIndexer<TodoIndexEntry, Integer, FileContent> getTodoIndexer(
FileType fileType, final VirtualFile virtualFile) {
final DataIndexer<TodoIndexEntry, Integer, FileContent> extIndexer;
if (fileType instanceof SubstitutedFileType
&& !((SubstitutedFileType) fileType).isSameFileType()) {
SubstitutedFileType sft = (SubstitutedFileType) fileType;
extIndexer =
new CompositeTodoIndexer(
getTodoIndexer(sft.getOriginalFileType(), virtualFile),
getTodoIndexer(sft.getFileType(), virtualFile));
} else {
extIndexer = TodoIndexers.INSTANCE.forFileType(fileType);
}
if (extIndexer != null) {
return extIndexer;
}
if (fileType instanceof LanguageFileType) {
final Language lang = ((LanguageFileType) fileType).getLanguage();
final ParserDefinition parserDef = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
final TokenSet commentTokens = parserDef != null ? parserDef.getCommentTokens() : null;
if (commentTokens != null) {
return new TokenSetTodoIndexer(commentTokens, virtualFile);
}
}
if (fileType instanceof CustomSyntaxTableFileType) {
return new TokenSetTodoIndexer(ABSTRACT_FILE_COMMENT_TOKENS, virtualFile);
}
return null;
}
/**
* Tries to calculate given line's indent column assuming that there might be a comment at the
* given indent offset (see {@link #getCommentPrefix(IElementType)}).
*
* @param line target line
* @param indentOffset start indent offset to use for the given line
* @param lineEndOffset given line's end offset
* @param fallbackColumn column to return if it's not possible to apply comment-specific indent
* calculation rules
* @return given line's indent column to use
*/
private int calcIndent(int line, int indentOffset, int lineEndOffset, int fallbackColumn) {
final HighlighterIterator it = myEditor.getHighlighter().createIterator(indentOffset);
IElementType tokenType = it.getTokenType();
Language language = tokenType.getLanguage();
TokenSet comments = myComments.get(language);
if (comments == null) {
ParserDefinition definition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
if (definition != null) {
comments = definition.getCommentTokens();
}
if (comments == null) {
return fallbackColumn;
} else {
myComments.put(language, comments);
}
}
if (comments.contains(tokenType) && indentOffset == it.getStart()) {
String prefix = COMMENT_PREFIXES.get(tokenType);
if (prefix == null) {
prefix = getCommentPrefix(tokenType);
}
if (!NO_COMMENT_INFO_MARKER.equals(prefix)) {
final int indentInsideCommentOffset =
CharArrayUtil.shiftForward(
myChars, indentOffset + prefix.length(), lineEndOffset, " \t");
if (indentInsideCommentOffset < lineEndOffset) {
int indent = myEditor.calcColumnNumber(indentInsideCommentOffset, line);
indentAfterUncomment.put(line, indent - prefix.length());
return indent;
}
}
}
return fallbackColumn;
}
@Override
@Nullable
protected PsiFile createFile(@NotNull final Language lang) {
if (lang == getTemplateDataLanguage()) {
final PsiFileImpl file =
(PsiFileImpl)
LanguageParserDefinitions.INSTANCE.forLanguage(StdLanguages.HTML).createFile(this);
file.setContentElementType(EmbeddedPerlTokens.HTML_TEMPLATE_DATA);
return file;
}
if (lang == getBaseLanguage()) {
return LanguageParserDefinitions.INSTANCE.forLanguage(lang).createFile(this);
}
return null;
}
@Nullable
protected PsiFile createFile(Language lang) {
if (lang != getBaseLanguage()) return null;
final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
if (parserDefinition != null) {
return parserDefinition.createFile(this);
}
return null;
}
public static boolean isInStringLiteral(PsiElement element) {
ParserDefinition definition =
LanguageParserDefinitions.INSTANCE.forLanguage(
PsiUtilBase.findLanguageFromElement(element));
if (definition == null) {
return false;
}
return isStringLiteral(element, definition)
|| isStringLiteral(element.getParent(), definition)
|| isStringLiteralWithError(element, definition)
|| isStringLiteralWithError(element.getParent(), definition);
}
private static void initState(IElementType root, PsiBuilder builder, ErrorState state) {
PsiFile file = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY);
state.completionState = file == null ? null : file.getUserData(COMPLETION_STATE_KEY);
Language language = file == null ? root.getLanguage() : file.getLanguage();
state.caseSensitive = language.isCaseSensitive();
ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
if (parserDefinition != null) {
state.commentTokens = parserDefinition.getCommentTokens();
state.whitespaceTokens = parserDefinition.getWhitespaceTokens();
}
PairedBraceMatcher matcher = LanguageBraceMatching.INSTANCE.forLanguage(language);
state.braces = matcher == null ? null : matcher.getPairs();
if (state.braces != null && state.braces.length == 0) state.braces = null;
}
@Override
public ASTNode parseContents(ASTNode chameleon) {
final CharTable table = SharedImplUtil.findCharTableByTree(chameleon);
final FileElement treeElement =
new DummyHolder(((TreeElement) chameleon).getManager(), null, table).getTreeElement();
final PsiFile file = (PsiFile) TreeUtil.getFileElement((TreeElement) chameleon).getPsi();
PsiFile originalFile = file.getOriginalFile();
final TemplateLanguageFileViewProvider viewProvider =
(TemplateLanguageFileViewProvider) originalFile.getViewProvider();
final Language language = getTemplateFileLanguage(viewProvider);
final CharSequence chars = chameleon.getChars();
final PsiFile templateFile = createTemplateFile(file, language, chars, viewProvider);
final TreeElement parsed = ((PsiFileImpl) templateFile).calcTreeElement();
Lexer langLexer =
LanguageParserDefinitions.INSTANCE.forLanguage(language).createLexer(file.getProject());
final Lexer lexer =
new MergingLexerAdapter(
new TemplateBlackAndWhiteLexer(
createBaseLexer(viewProvider),
langLexer,
myTemplateElementType,
myOuterElementType),
TokenSet.create(myTemplateElementType, myOuterElementType));
lexer.start(chars);
insertOuters(parsed, lexer, table);
if (parsed != null) {
final TreeElement element = parsed.getFirstChildNode();
if (element != null) {
((CompositeElement) parsed).rawRemoveAllChildren();
treeElement.rawAddChildren(element);
}
}
treeElement.subtreeChanged();
TreeElement childNode = treeElement.getFirstChildNode();
DebugUtil.checkTreeStructure(parsed);
DebugUtil.checkTreeStructure(treeElement);
DebugUtil.checkTreeStructure(chameleon);
DebugUtil.checkTreeStructure(file.getNode());
DebugUtil.checkTreeStructure(originalFile.getNode());
return childNode;
}
private static boolean isLiteral(PsiElement element) {
if (element == null) return false;
final ASTNode astNode = element.getNode();
if (astNode == null) {
return false;
}
final IElementType elementType = astNode.getElementType();
final ParserDefinition parserDefinition =
LanguageParserDefinitions.INSTANCE.forLanguage(element.getLanguage());
if (parserDefinition != null) {
final TokenSet literals = parserDefinition.getStringLiteralElements();
return literals.contains(elementType);
}
return false;
}
public static boolean isCommentToken(
@NotNull IElementType tokenType, @NotNull LanguageVersion languageVersion) {
final Language language = tokenType.getLanguage();
if (language != languageVersion.getLanguage()) {
return false;
}
boolean inComments = false;
final ParserDefinition parserDefinition =
LanguageParserDefinitions.INSTANCE.forLanguage(language);
if (parserDefinition != null) {
final TokenSet commentTokens = parserDefinition.getCommentTokens(languageVersion);
if (commentTokens.contains(tokenType)) {
inComments = true;
}
}
return inComments;
}
public static boolean canHaveStub(@NotNull VirtualFile file) {
final FileType fileType = file.getFileType();
if (fileType instanceof LanguageFileType) {
Language l = ((LanguageFileType) fileType).getLanguage();
ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
if (parserDefinition == null) return false;
final IFileElementType elementType = parserDefinition.getFileNodeType();
return elementType instanceof IStubFileElementType
&& (((IStubFileElementType) elementType).shouldBuildStubFor(file)
|| IndexingStamp.isFileIndexed(
file, INDEX_ID, IndexInfrastructure.getIndexCreationStamp(INDEX_ID)));
}
if (fileType.isBinary()) {
final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
return builder != null && builder.acceptsFile(file);
}
return false;
}
private void doVisitElement(PsiElement element) {
CompiledPattern pattern = myGlobalVisitor.getContext().getPattern();
if (myGlobalVisitor.getCodeBlockLevel() == 0) {
initTopLevelElement(element);
return;
}
if (canBePatternVariable(element) && pattern.isRealTypedVar(element)) {
myGlobalVisitor.handle(element);
final MatchingHandler handler = pattern.getHandler(element);
handler.setFilter(
new NodeFilter() {
public boolean accepts(PsiElement other) {
return canBePatternVariableValue(other);
}
});
super.visitElement(element);
return;
}
super.visitElement(element);
if (myGlobalVisitor.getContext().getSearchHelper().doOptimizing()
&& element instanceof LeafElement) {
ParserDefinition parserDefinition =
LanguageParserDefinitions.INSTANCE.forLanguage(element.getLanguage());
if (parserDefinition != null) {
String text = element.getText();
// todo: support variables inside comments
boolean flag = true;
if (StringUtil.isJavaIdentifier(text) && flag) {
myGlobalVisitor.processTokenizedName(
text, true, GlobalCompilingVisitor.OccurenceKind.CODE);
}
}
}
}
@Override
public boolean acceptInput(final VirtualFile file) {
if (!(file.getFileSystem() instanceof LocalFileSystem)) {
return false; // do not index TODOs in library sources
}
final FileType fileType = file.getFileType();
if (ProjectUtil.isProjectOrWorkspaceFile(file, fileType)) {
return false;
}
if (fileType instanceof LanguageFileType) {
final Language lang = ((LanguageFileType) fileType).getLanguage();
final ParserDefinition parserDef = LanguageParserDefinitions.INSTANCE.forLanguage(lang);
final TokenSet commentTokens = parserDef != null ? parserDef.getCommentTokens() : null;
return commentTokens != null;
}
return IdTableBuilding.isTodoIndexerRegistered(fileType)
|| fileType instanceof AbstractFileType;
}
private static int getCumulativeVersion() {
int version = VERSION;
for (final FileType fileType : FileTypeManager.getInstance().getRegisteredFileTypes()) {
if (fileType instanceof LanguageFileType) {
Language l = ((LanguageFileType) fileType).getLanguage();
ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
if (parserDefinition != null) {
final IFileElementType type = parserDefinition.getFileNodeType();
if (type instanceof IStubFileElementType) {
version += ((IStubFileElementType) type).getStubVersion();
}
}
} else if (fileType.isBinary()) {
final BinaryFileStubBuilder builder = BinaryFileStubBuilders.INSTANCE.forFileType(fileType);
if (builder != null) {
version += builder.getStubVersion();
}
}
}
return version;
}
@NotNull
@Override
public PsiElement createDummyHolder(
@NotNull final String text,
@NotNull final IElementType type,
@Nullable final PsiElement context) {
final DummyHolder result = DummyHolderFactory.createHolder(myManager, context);
final FileElement holder = result.getTreeElement();
final Language language = type.getLanguage();
final ParserDefinition parserDefinition =
LanguageParserDefinitions.INSTANCE.forLanguage(language);
assert parserDefinition != null : "No parser definition for language " + language;
final Project project = myManager.getProject();
final Lexer lexer = parserDefinition.createLexer(project);
final PsiBuilder builder =
PsiBuilderFactory.getInstance().createBuilder(project, holder, lexer, language, text);
final ASTNode node = parserDefinition.createParser(project).parse(type, builder);
holder.rawAddChildren((TreeElement) node);
final PsiElement psi = node.getPsi();
assert psi != null : text;
return psi;
}
protected SpacingBuilder createSpacingBuilder(CodeStyleSettings settings) {
final IFileElementType file =
LanguageParserDefinitions.INSTANCE
.forLanguage(PythonLanguage.getInstance())
.getFileNodeType();
final PyCodeStyleSettings pySettings = settings.getCustomSettings(PyCodeStyleSettings.class);
final CommonCodeStyleSettings commonSettings =
settings.getCommonSettings(PythonLanguage.getInstance());
return new SpacingBuilder(commonSettings)
.between(CLASS_DECLARATION, STATEMENT_OR_DECLARATION)
.blankLines(commonSettings.BLANK_LINES_AROUND_CLASS)
.between(STATEMENT_OR_DECLARATION, CLASS_DECLARATION)
.blankLines(commonSettings.BLANK_LINES_AROUND_CLASS)
.between(FUNCTION_DECLARATION, STATEMENT_OR_DECLARATION)
.blankLines(commonSettings.BLANK_LINES_AROUND_METHOD)
.between(STATEMENT_OR_DECLARATION, FUNCTION_DECLARATION)
.blankLines(commonSettings.BLANK_LINES_AROUND_METHOD)
.after(FUNCTION_DECLARATION)
.blankLines(commonSettings.BLANK_LINES_AROUND_METHOD)
.after(CLASS_DECLARATION)
.blankLines(commonSettings.BLANK_LINES_AROUND_CLASS)
// Remove excess blank lines between imports, because ImportOptimizer gets rid of them
// anyway.
// Empty lines between import groups are handles in PyBlock#getSpacing
.between(IMPORT_STATEMENTS, IMPORT_STATEMENTS)
.spacing(0, Integer.MAX_VALUE, 1, false, 0)
.between(STATEMENT_OR_DECLARATION, STATEMENT_OR_DECLARATION)
.spacing(0, Integer.MAX_VALUE, 1, false, 1)
.between(COLON, STATEMENT_LIST)
.spacing(1, Integer.MAX_VALUE, 0, true, 0)
.afterInside(COLON, TokenSet.create(KEY_VALUE_EXPRESSION, LAMBDA_EXPRESSION))
.spaceIf(pySettings.SPACE_AFTER_PY_COLON)
.afterInside(GT, ANNOTATION)
.spaces(1)
.betweenInside(MINUS, GT, ANNOTATION)
.none()
.beforeInside(ANNOTATION, FUNCTION_DECLARATION)
.spaces(1)
.beforeInside(ANNOTATION, NAMED_PARAMETER)
.none()
.afterInside(COLON, ANNOTATION)
.spaces(1)
.afterInside(RARROW, ANNOTATION)
.spaces(1)
.between(allButLambda(), PARAMETER_LIST)
.spaceIf(commonSettings.SPACE_BEFORE_METHOD_PARENTHESES)
.before(COLON)
.spaceIf(pySettings.SPACE_BEFORE_PY_COLON)
.after(COMMA)
.spaceIf(commonSettings.SPACE_AFTER_COMMA)
.before(COMMA)
.spaceIf(commonSettings.SPACE_BEFORE_COMMA)
.between(FROM_KEYWORD, DOT)
.spaces(1)
.between(DOT, IMPORT_KEYWORD)
.spaces(1)
.around(DOT)
.spaces(0)
.aroundInside(AT, DECORATOR_CALL)
.none()
.before(SEMICOLON)
.spaceIf(commonSettings.SPACE_BEFORE_SEMICOLON)
.withinPairInside(LPAR, RPAR, ARGUMENT_LIST)
.spaceIf(commonSettings.SPACE_WITHIN_METHOD_CALL_PARENTHESES)
.withinPairInside(LPAR, RPAR, PARAMETER_LIST)
.spaceIf(commonSettings.SPACE_WITHIN_METHOD_PARENTHESES)
.withinPairInside(LPAR, RPAR, FROM_IMPORT_STATEMENT)
.spaces(0)
.withinPairInside(LPAR, RPAR, GENERATOR_EXPRESSION)
.spaces(0)
.withinPairInside(LPAR, RPAR, PARENTHESIZED_EXPRESSION)
.spaces(0)
.before(LBRACKET)
.spaceIf(pySettings.SPACE_BEFORE_LBRACKET)
.afterInside(LBRACE, DICT_LITERAL_EXPRESSION)
.spaceIf(pySettings.SPACE_WITHIN_BRACES, pySettings.DICT_NEW_LINE_AFTER_LEFT_BRACE)
.beforeInside(RBRACE, DICT_LITERAL_EXPRESSION)
.spaceIf(pySettings.SPACE_WITHIN_BRACES, pySettings.DICT_NEW_LINE_BEFORE_RIGHT_BRACE)
.withinPair(LBRACE, RBRACE)
.spaceIf(pySettings.SPACE_WITHIN_BRACES)
.withinPair(LBRACKET, RBRACKET)
.spaceIf(commonSettings.SPACE_WITHIN_BRACKETS)
.before(ARGUMENT_LIST)
.spaceIf(commonSettings.SPACE_BEFORE_METHOD_CALL_PARENTHESES)
.around(DECORATOR_CALL)
.spacing(1, Integer.MAX_VALUE, 0, true, 0)
.after(DECORATOR_LIST)
.spacing(1, Integer.MAX_VALUE, 0, true, 0)
.aroundInside(EQ, ASSIGNMENT_STATEMENT)
.spaceIf(commonSettings.SPACE_AROUND_ASSIGNMENT_OPERATORS)
.aroundInside(EQ, NAMED_PARAMETER)
.spaceIf(pySettings.SPACE_AROUND_EQ_IN_NAMED_PARAMETER)
.aroundInside(EQ, KEYWORD_ARGUMENT_EXPRESSION)
.spaceIf(pySettings.SPACE_AROUND_EQ_IN_KEYWORD_ARGUMENT)
.around(AUG_ASSIGN_OPERATIONS)
.spaceIf(commonSettings.SPACE_AROUND_ASSIGNMENT_OPERATORS)
.aroundInside(ADDITIVE_OPERATIONS, BINARY_EXPRESSION)
.spaceIf(commonSettings.SPACE_AROUND_ADDITIVE_OPERATORS)
.aroundInside(MULTIPLICATIVE_OR_EXP, STAR_PARAMETERS)
.none()
.around(MULTIPLICATIVE_OR_EXP)
.spaceIf(commonSettings.SPACE_AROUND_MULTIPLICATIVE_OPERATORS)
.around(SHIFT_OPERATIONS)
.spaceIf(commonSettings.SPACE_AROUND_SHIFT_OPERATORS)
.around(BITWISE_OPERATIONS)
.spaceIf(commonSettings.SPACE_AROUND_BITWISE_OPERATORS)
.around(EQUALITY_OPERATIONS)
.spaceIf(commonSettings.SPACE_AROUND_EQUALITY_OPERATORS)
.around(RELATIONAL_OPERATIONS)
.spaceIf(commonSettings.SPACE_AROUND_RELATIONAL_OPERATORS)
.around(SINGLE_SPACE_KEYWORDS)
.spaces(1);
}
protected Lexer createBaseLexer(TemplateLanguageFileViewProvider viewProvider) {
return LanguageParserDefinitions.INSTANCE
.forLanguage(viewProvider.getBaseLanguage())
.createLexer(viewProvider.getManager().getProject());
}
private static boolean isCommentComplete(
PsiComment comment, CodeDocumentationAwareCommenter commenter, Editor editor) {
for (CommentCompleteHandler handler :
Extensions.getExtensions(CommentCompleteHandler.EP_NAME)) {
if (handler.isApplicable(comment, commenter)) {
return handler.isCommentComplete(comment, commenter, editor);
}
}
String commentText = comment.getText();
final boolean docComment = isDocComment(comment, commenter);
final String expectedCommentEnd =
docComment ? commenter.getDocumentationCommentSuffix() : commenter.getBlockCommentSuffix();
if (!commentText.endsWith(expectedCommentEnd)) return false;
final PsiFile containingFile = comment.getContainingFile();
final Language language = containingFile.getLanguage();
ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
if (parserDefinition == null) {
return true;
}
Lexer lexer = parserDefinition.createLexer(containingFile.getProject());
final String commentPrefix =
docComment ? commenter.getDocumentationCommentPrefix() : commenter.getBlockCommentPrefix();
lexer.start(
commentText, commentPrefix == null ? 0 : commentPrefix.length(), commentText.length());
QuoteHandler fileTypeHandler = TypedHandler.getQuoteHandler(containingFile, editor);
JavaLikeQuoteHandler javaLikeQuoteHandler =
fileTypeHandler instanceof JavaLikeQuoteHandler
? (JavaLikeQuoteHandler) fileTypeHandler
: null;
while (true) {
IElementType tokenType = lexer.getTokenType();
if (tokenType == null) {
return false;
}
if (javaLikeQuoteHandler != null
&& javaLikeQuoteHandler.getStringTokenTypes() != null
&& javaLikeQuoteHandler.getStringTokenTypes().contains(tokenType)) {
String text = commentText.substring(lexer.getTokenStart(), lexer.getTokenEnd());
int endOffset = comment.getTextRange().getEndOffset();
if (text.endsWith(expectedCommentEnd)
&& endOffset < containingFile.getTextLength()
&& containingFile.getText().charAt(endOffset) == '\n') {
return true;
}
}
if (tokenType == commenter.getDocumentationCommentTokenType()
|| tokenType == commenter.getBlockCommentTokenType()) {
return false;
}
if (tokenType == commenter.getLineCommentTokenType()
&& lexer.getTokenText().contains(commentPrefix)) {
return false;
}
if (lexer.getTokenEnd() == commentText.length()) {
if (tokenType == commenter.getLineCommentTokenType()) {
String prefix = commenter.getLineCommentPrefix();
lexer.start(
commentText,
lexer.getTokenStart() + (prefix == null ? 0 : prefix.length()),
commentText.length());
lexer.advance();
continue;
} else if (isInvalidPsi(comment)) {
return false;
}
return true;
}
lexer.advance();
}
}