public void parseSliceEnd(PsiBuilder.Marker exprStart, PsiBuilder.Marker sliceItemStart) { myBuilder.advanceLexer(); if (atToken(PyTokenTypes.RBRACKET)) { PsiBuilder.Marker sliceMarker = myBuilder.mark(); sliceMarker.done(PyElementTypes.EMPTY_EXPRESSION); sliceItemStart.done(PyElementTypes.SLICE_ITEM); nextToken(); exprStart.done(PyElementTypes.SLICE_EXPRESSION); return; } else { if (atToken(PyTokenTypes.COLON)) { PsiBuilder.Marker sliceMarker = myBuilder.mark(); sliceMarker.done(PyElementTypes.EMPTY_EXPRESSION); } else { parseSingleExpression(false); } if (!BRACKET_COLON_COMMA.contains(myBuilder.getTokenType())) { myBuilder.error(message("PARSE.expected.colon.or.rbracket")); } if (matchToken(PyTokenTypes.COLON)) { parseSingleExpression(false); } sliceItemStart.done(PyElementTypes.SLICE_ITEM); if (!BRACKET_OR_COMMA.contains(myBuilder.getTokenType())) { myBuilder.error("']' or ',' expected"); } } parseSliceListTail(exprStart, null); }
/** * Named list operators * * @param b PerlBuilder * @param l Parsing level * @return parsing result */ public static boolean isListOperator(PsiBuilder b, int l) { PerlTokenData prevTokenData = ((PerlBuilder) b).lookupToken(-1); if (prevTokenData != null && prevTokenData.getTokenType() == OPERATOR_DEREFERENCE) return false; IElementType tokenType = b.getTokenType(); IElementType nextTokenType = b.lookAhead(1); if (CONVERTABLE_TOKENS.contains(tokenType) && nextTokenType != LEFT_PAREN // not function call && !PACKAGE_TOKENS.contains(nextTokenType) // not method Package:: && !(nextTokenType == IDENTIFIER && ((PerlBuilder) b) .isKnownPackage( ((PerlBuilder) b).lookupToken(1).getTokenText())) // not Method Package ) // todo we should check current namespace here return !PerlSubUtil.BUILT_IN_UNARY.contains(b.getTokenText()); else if (PACKAGE_TOKENS.contains(tokenType) && CONVERTABLE_TOKENS.contains(nextTokenType) && b.lookAhead(2) != LEFT_PAREN) return !PerlSubUtil.isUnary( b.getTokenText(), ((PerlBuilder) b).lookupToken(1).getTokenText()); return false; }
/** * Statement recovery function. Should not consume token, only check; * * @param b PerlBuilder * @param l parsing level * @return parsing result */ public static boolean recoverStatement(PsiBuilder b, int l) { assert b instanceof PerlBuilder; if (!((PerlBuilder) b).isRecoveringStatement()) ((PerlBuilder) b).startRecovery(); IElementType currentTokenType = b.getTokenType(); // System.err.println("Checking " + b.getTokenText() + currentTokenType); if (currentTokenType == null // got end of file || ((PerlBuilder) b).getBracesLevel() == 0 && ( // we are not in braced statement UNCONDITIONAL_STATEMENT_RECOVERY_TOKENS.contains( currentTokenType) // got semi, package, end of regex, use, compound or suffix || currentTokenType == RESERVED_SUB && STATEMENT_RECOVERY_SUB_SUFFIX.contains(b.lookAhead(1)) // got sub definition )) { ((PerlBuilder) b).stopRecovery(); return false; } if (currentTokenType == LEFT_BRACE) ((PerlBuilder) b).openBrace(); else if (currentTokenType == RIGHT_BRACE) ((PerlBuilder) b).closeBrace(); return true; }
boolean typeExtends(IElementType child, IElementType parent) { if (child == parent) return true; if (extendsSets != null) { for (TokenSet set : extendsSets) { if (set.contains(child) && set.contains(parent)) return true; } } return altExtendsChecker != null && altExtendsChecker.process(child, parent); }
@Nullable private PsiBuilder.Marker parseUnary(final PsiBuilder builder) { final IElementType tokenType = builder.getTokenType(); if (PREFIX_OPS.contains(tokenType)) { final PsiBuilder.Marker unary = builder.mark(); builder.advanceLexer(); final PsiBuilder.Marker operand = parseUnary(builder); if (operand == null) { error(builder, JavaErrorMessages.message("expected.expression")); } unary.done(JavaElementType.PREFIX_EXPRESSION); return unary; } else if (tokenType == JavaTokenType.LPARENTH) { final PsiBuilder.Marker typeCast = builder.mark(); builder.advanceLexer(); ReferenceParser.TypeInfo typeInfo = myParser .getReferenceParser() .parseTypeInfo( builder, ReferenceParser.EAT_LAST_DOT | ReferenceParser.WILDCARD | ReferenceParser.CONJUNCTIONS | ReferenceParser.INCOMPLETE_ANNO); if (typeInfo == null || !expect(builder, JavaTokenType.RPARENTH)) { typeCast.rollbackTo(); return parsePostfix(builder); } if (PREF_ARITHMETIC_OPS.contains(builder.getTokenType()) && !typeInfo.isPrimitive) { typeCast.rollbackTo(); return parsePostfix(builder); } final PsiBuilder.Marker expr = parseUnary(builder); if (expr == null) { if (!typeInfo .isParameterized) { // cannot parse correct parenthesized expression after correct // parameterized type typeCast.rollbackTo(); return parsePostfix(builder); } else { error(builder, JavaErrorMessages.message("expected.expression")); } } typeCast.done(JavaElementType.TYPE_CAST_EXPRESSION); return typeCast; } else { return parsePostfix(builder); } }
/** * There is a possible case that given base node doesn't have {@link * AlignmentInColumnsConfig#getTargetDeclarationTypes() target type} but its first child node or * first child node of the first child node etc does. * * <p>This method tries to derive node of the target type from the given node. * * @param baseNode base node to process * @param targetTypes target node types * @return base node or its first descendant child that has {@link * AlignmentInColumnsConfig#getTargetDeclarationTypes() target type} target type if the one if * found; <code>null</code> otherwise */ @Nullable private static ASTNode deriveNodeOfTargetType(ASTNode baseNode, TokenSet targetTypes) { if (targetTypes.contains(baseNode.getElementType())) { return baseNode; } for (ASTNode node = baseNode; node != null; node = node.getFirstChildNode()) { IElementType nodeType = node.getElementType(); if (targetTypes.contains(nodeType)) { return node; } } return null; }
/** * Tries to calculate given line's indent column assuming that there might be a comment at the * given indent offset (see {@link #getCommentPrefix(IElementType)}). * * @param line target line * @param indentOffset start indent offset to use for the given line * @param lineEndOffset given line's end offset * @param fallbackColumn column to return if it's not possible to apply comment-specific indent * calculation rules * @return given line's indent column to use */ private int calcIndent(int line, int indentOffset, int lineEndOffset, int fallbackColumn) { final HighlighterIterator it = myEditor.getHighlighter().createIterator(indentOffset); IElementType tokenType = it.getTokenType(); Language language = tokenType.getLanguage(); TokenSet comments = myComments.get(language); if (comments == null) { ParserDefinition definition = LanguageParserDefinitions.INSTANCE.forLanguage(language); if (definition != null) { comments = definition.getCommentTokens(); } if (comments == null) { return fallbackColumn; } else { myComments.put(language, comments); } } if (comments.contains(tokenType) && indentOffset == it.getStart()) { String prefix = COMMENT_PREFIXES.get(tokenType); if (prefix == null) { prefix = getCommentPrefix(tokenType); } if (!NO_COMMENT_INFO_MARKER.equals(prefix)) { final int indentInsideCommentOffset = CharArrayUtil.shiftForward( myChars, indentOffset + prefix.length(), lineEndOffset, " \t"); if (indentInsideCommentOffset < lineEndOffset) { int indent = myEditor.calcColumnNumber(indentInsideCommentOffset, line); indentAfterUncomment.put(line, indent - prefix.length()); return indent; } } } return fallbackColumn; }
void addNestedChildrenSuffix( List<Block> list, @Nullable AlignmentProvider.Aligner aligner, boolean topLevel, List<ASTNode> children, int limit) { for (int i = 1; i < limit; i++) { ASTNode childNode = children.get(i); if (canBeCorrectBlock(childNode)) { IElementType type = childNode.getElementType(); Indent indent = topLevel || NESTED.contains(type) || type == mIDENT || TokenSets.DOTS.contains(type) ? Indent.getContinuationWithoutFirstIndent() : Indent.getNoneIndent(); if (aligner != null && TokenSets.DOTS.contains(type)) { aligner.append(childNode.getPsi()); } list.add( new GroovyBlock( childNode, indent, myWrap, mySettings, myGroovySettings, myAlignmentProvider)); } } }
@Nullable private static PsiBuilder.Marker parseBinary( final PsiBuilder builder, final ExprType type, final TokenSet ops) { PsiBuilder.Marker left = parseExpression(builder, type); if (left == null) return null; while (true) { final IElementType tokenType = getGtTokenType(builder); if (tokenType == null || !ops.contains(tokenType)) break; final PsiBuilder.Marker binary = left.precede(); advanceGtToken(builder, tokenType); final PsiBuilder.Marker right = parseExpression(builder, type); if (right == null) { error(builder, JavaErrorMessages.message("expected.expression")); binary.done(JavaElementType.BINARY_EXPRESSION); return binary; } binary.done(JavaElementType.BINARY_EXPRESSION); left = binary; } return left; }
private boolean parseSliceListTail( PsiBuilder.Marker exprStart, @Nullable PsiBuilder.Marker sliceOrTupleStart) { boolean inSlice = sliceOrTupleStart == null; while (atToken(PyTokenTypes.COMMA)) { nextToken(); PsiBuilder.Marker sliceItemStart = myBuilder.mark(); parseTestExpression(false, false); if (matchToken(PyTokenTypes.COLON)) { inSlice = true; parseTestExpression(false, false); if (matchToken(PyTokenTypes.COLON)) { parseTestExpression(false, false); } } sliceItemStart.done(PyElementTypes.SLICE_ITEM); if (!BRACKET_OR_COMMA.contains(myBuilder.getTokenType())) { myBuilder.error("']' or ',' expected"); break; } } checkMatches(PyTokenTypes.RBRACKET, message("PARSE.expected.rbracket")); if (inSlice) { if (sliceOrTupleStart != null) { sliceOrTupleStart.drop(); } exprStart.done(PyElementTypes.SLICE_EXPRESSION); } return inSlice; }
@NotNull private static Indent indentIfNotBrace(@NotNull ASTNode child) { if (BRACES_TOKEN_SET.contains(child.getElementType())) { return Indent.getNoneIndent(); } else { return Indent.getNormalIndent(); } }
@NotNull public static PsiBuilder.Marker parseArgumentList(final PsiBuilder builder) { final PsiBuilder.Marker list = builder.mark(); builder.advanceLexer(); boolean first = true; while (true) { final IElementType tokenType = builder.getTokenType(); if (first && (ARGS_LIST_END.contains(tokenType) || builder.eof())) break; if (!first && !ARGS_LIST_CONTINUE.contains(tokenType)) break; boolean hasError = false; if (!first) { if (builder.getTokenType() == JavaTokenType.COMMA) { builder.advanceLexer(); } else { hasError = true; error(builder, JavaErrorMessages.message("expected.comma.or.rparen")); emptyExpression(builder); } } first = false; final PsiBuilder.Marker arg = parse(builder); if (arg == null) { if (!hasError) { error(builder, JavaErrorMessages.message("expected.expression")); emptyExpression(builder); } if (!ARGS_LIST_CONTINUE.contains(builder.getTokenType())) break; if (builder.getTokenType() != JavaTokenType.COMMA && !builder.eof()) { builder.advanceLexer(); } } } final boolean closed = JavaParserUtil.expectOrError( builder, JavaTokenType.RPARENTH, JavaErrorMessages.message("expected.rparen")); list.done(JavaElementType.EXPRESSION_LIST); if (!closed) { list.setCustomEdgeTokenBinders(null, GREEDY_RIGHT_EDGE_PROCESSOR); } return list; }
public static boolean isPrecededBy( @Nullable ASTNode node, TokenSet expectedTypes, IElementType... skipTypes) { ASTNode prevNode = node == null ? null : node.getTreePrev(); while (prevNode != null && (isWhitespaceOrEmpty(prevNode) || isOneOf(prevNode, skipTypes))) { prevNode = prevNode.getTreePrev(); } if (prevNode == null) return false; return expectedTypes.contains(prevNode.getElementType()); }
/** * parser for print/say/printf filehandle * * @param b PerlBuilder * @param l parsing level * @return parsing result */ public static boolean parsePrintHandle(PsiBuilder b, int l) { IElementType currentTokenType = b.getTokenType(); IElementType nextTokenType = b.lookAhead(1); assert b instanceof PerlBuilder; if (CONVERTABLE_TOKENS.contains(currentTokenType) // it's identifier && !PRINT_HANDLE_NEGATE_SUFFIX.contains(nextTokenType) // no negation tokens && !PerlSubUtil.BUILT_IN.contains( b.getTokenText()) // it's not built in. crude, probably we should check any known sub ) { PsiBuilder.Marker m = b.mark(); b.advanceLexer(); m.collapse(HANDLE); return true; } return false; }
@Nullable public static ASTNode skipWhitespaceCommentsAndTokens(final ASTNode node, TokenSet alsoSkip) { ASTNode element = node; while (true) { if (element == null) return null; if (!isWhitespaceOrComment(element) && !alsoSkip.contains(element.getElementType())) break; element = element.getTreeNext(); } return element; }
@NotNull private Indent calcIndent(@NotNull ASTNode child) { IElementType parentType = myNode.getElementType(); if (parentType == TOP_LEVEL) return Indent.getAbsoluteNoneIndent(); if (BLOCKS_TOKEN_SET.contains(parentType)) { return indentIfNotBrace(child); } else { return Indent.getNoneIndent(); } }
/** * Parsing label declaration LABEL: * * @param b PerlBuilder * @param l parsing level * @return parsing result */ public static boolean parseLabelDeclaration(PsiBuilder b, int l) { if (CONVERTABLE_TOKENS.contains(b.getTokenType()) && b.lookAhead(1) == COLON) { PsiBuilder.Marker m = b.mark(); b.advanceLexer(); m.collapse(LABEL); b.advanceLexer(); return true; } return false; }
/** * Checks for version token and convert if necessary * * @param b PerlBuilder * @param l parsing level * @return parsing result */ public static boolean parsePerlVersion(PsiBuilder b, int l) { if (b.getTokenType() == NUMBER_VERSION) return consumeToken(b, NUMBER_VERSION); else if (VERSION_TOKENS.contains(b.getTokenType())) { PsiBuilder.Marker m = b.mark(); b.advanceLexer(); m.collapse(NUMBER_VERSION); return true; } return false; }
public GoBinaryExpressionBlock( ASTNode node, Alignment alignment, Wrap wrap, CommonCodeStyleSettings settings) { super(node, alignment, Indent.getNormalIndent(), wrap, settings); GoBinaryExpression psi = node.getPsi(GoBinaryExpression.class); if (psi != null && EMPTY_SPACE_TOKENS.contains(psi.getOperator())) { spacing = EMPTY_SPACING_KEEP_LINE_BREAKS; } else { spacing = BASIC_SPACING_KEEP_LINE_BREAKS; } }
@Nullable protected Indent getChildIndent() { IElementType parentType = myNode.getElementType(); // always the parent since isIncomplete is false if (BLOCKS_TOKEN_SET.contains(parentType)) { return Indent.getNormalIndent(); } else { return Indent.getNoneIndent(); } }
public static boolean convertBracedString(PsiBuilder b, int l) { if (CONVERTABLE_TOKENS.contains(b.getTokenType()) && b.lookAhead(1) == RIGHT_BRACE) { // fixme shouldn't we add string_sq here? PsiBuilder.Marker m = b.mark(); b.advanceLexer(); m.collapse(STRING_CONTENT); return true; } return false; }
// @todo this is really raw public static boolean parseVariableAttributes(PsiBuilder b, int l) { PsiBuilder.Marker m = null; while (!b.eof() && !VARIABLE_ATTRIBUTE_STOP_TOKENS.contains(b.getTokenType())) { if (m == null) m = b.mark(); b.advanceLexer(); } if (m != null) m.collapse(VAR_ATTRIBUTE); return true; }
private static List<IElementType> findSubNodeTypes(ASTNode node, TokenSet types) { List<IElementType> foundTypes = new SmartList<IElementType>(); for (ASTNode child = node.getFirstChildNode(); child != null && child.getTreeParent() == node; child = child.getTreeNext()) { IElementType type = child.getElementType(); if (types.contains(type)) { foundTypes.add(type); } } return foundTypes; }
/** * Checks and parses bareword filehandle for <FH> operations * * @param b PerlBuilder * @param l parsing level * @return parsing result */ public static boolean parseReadHandle(PsiBuilder b, int l) { IElementType currentTokenType = b.getTokenType(); IElementType nextTokenType = b.lookAhead(1); if (CONVERTABLE_TOKENS.contains(currentTokenType) && nextTokenType == OPERATOR_GT_NUMERIC) { PsiBuilder.Marker m = b.mark(); b.advanceLexer(); m.collapse(HANDLE); return true; } return false; }
@Nullable private PsiBuilder.Marker parseBinary( final PsiBuilder builder, final ExprType type, final TokenSet ops) { PsiBuilder.Marker result = parseExpression(builder, type); if (result == null) return null; int operandCount = 1; IElementType tokenType = getGtTokenType(builder); IElementType currentExprTokenType = tokenType; while (true) { if (tokenType == null || !ops.contains(tokenType)) break; advanceGtToken(builder, tokenType); final PsiBuilder.Marker right = parseExpression(builder, type); operandCount++; tokenType = getGtTokenType(builder); if (tokenType == null || !ops.contains(tokenType) || tokenType != currentExprTokenType || right == null) { // save result = result.precede(); if (right == null) { error(builder, JavaErrorMessages.message("expected.expression")); } result.done( operandCount > 2 ? JavaElementType.POLYADIC_EXPRESSION : JavaElementType.BINARY_EXPRESSION); if (right == null) break; currentExprTokenType = tokenType; operandCount = 1; } } return result; }
/** * Merges sequence [package] identifier to a package * * @param b PerlBuilder * @param l parsing level * @return result */ public static boolean mergePackageName(PsiBuilder b, int l) { IElementType tokenType = b.getTokenType(); if (tokenType == PACKAGE) { b.advanceLexer(); return true; } else if (PACKAGE_TOKENS.contains(tokenType) && CONVERTABLE_TOKENS.contains(b.lookAhead(1))) { PsiBuilder.Marker m = b.mark(); b.advanceLexer(); b.advanceLexer(); m.collapse(PACKAGE); return true; } else if (PACKAGE_TOKENS.contains(tokenType) // explicit package name, like Foo::->method() || CONVERTABLE_TOKENS.contains(tokenType) // single word package ) { PsiBuilder.Marker m = b.mark(); b.advanceLexer(); m.collapse(PACKAGE); return true; } return false; }
@Override @NotNull public Map<TodoIndexEntry, Integer> map(@NotNull final FileContent inputData) { if (IndexPatternUtil.getIndexPatternCount() > 0) { final CharSequence chars = inputData.getContentAsText(); final OccurrenceConsumer occurrenceConsumer = new OccurrenceConsumer(null, true); EditorHighlighter highlighter; final EditorHighlighter editorHighlighter = inputData.getUserData(EDITOR_HIGHLIGHTER); if (editorHighlighter != null && checkCanUseCachedEditorHighlighter(chars, editorHighlighter)) { highlighter = editorHighlighter; } else { highlighter = HighlighterFactory.createHighlighter(inputData.getProject(), myFile); highlighter.setText(chars); } final int documentLength = chars.length(); BaseFilterLexer.TodoScanningState todoScanningState = null; final HighlighterIterator iterator = highlighter.createIterator(0); while (!iterator.atEnd()) { final IElementType token = iterator.getTokenType(); if (myCommentTokens.contains(token) || CacheUtil.isInComments(token)) { int start = iterator.getStart(); if (start >= documentLength) break; int end = iterator.getEnd(); todoScanningState = BaseFilterLexer.advanceTodoItemsCount( chars.subSequence(start, Math.min(end, documentLength)), occurrenceConsumer, todoScanningState); if (end > documentLength) break; } iterator.advance(); } final Map<TodoIndexEntry, Integer> map = new HashMap<>(); for (IndexPattern pattern : IndexPatternUtil.getIndexPatterns()) { final int count = occurrenceConsumer.getOccurrenceCount(pattern); if (count > 0) { map.put( new TodoIndexEntry(pattern.getPatternString(), pattern.isCaseSensitive()), count); } } return map; } return Collections.emptyMap(); }
/** * Replaces identifier as a variable name * * @param b Perlbuilder * @param l parsing level * @return parsing result */ public static boolean convertIdentifier(PsiBuilder b, int l, IElementType tokenType) { IElementType currentTokenType = b.getTokenType(); if (currentTokenType == tokenType) { b.advanceLexer(); return true; } else if (CONVERTABLE_TOKENS.contains(currentTokenType)) { PsiBuilder.Marker m = b.mark(); b.advanceLexer(); m.collapse(tokenType); return true; } return false; }
/** * Completes namespace, invoked when we are 100% sure that PACKAGE_IDENTIFIER is a package * * @param b Perlbuilder * @param l parsing level * @return parsing result */ public static boolean convertPackageIdentifier(PsiBuilder b, int l) { IElementType currentTokenType = b.getTokenType(); if (currentTokenType == PACKAGE) { b.advanceLexer(); return true; } else if (PACKAGE_TOKENS.contains(currentTokenType)) { PsiBuilder.Marker m = b.mark(); b.advanceLexer(); m.collapse(PACKAGE); return true; } return false; }
@Nullable private PsiBuilder.Marker parsePostfix(final PsiBuilder builder) { PsiBuilder.Marker operand = parsePrimary(builder, null, -1); if (operand == null) return null; while (POSTFIX_OPS.contains(builder.getTokenType())) { final PsiBuilder.Marker postfix = operand.precede(); builder.advanceLexer(); postfix.done(JavaElementType.POSTFIX_EXPRESSION); operand = postfix; } return operand; }