public void parseSliceEnd(PsiBuilder.Marker exprStart, PsiBuilder.Marker sliceItemStart) {
myBuilder.advanceLexer();
if (atToken(PyTokenTypes.RBRACKET)) {
PsiBuilder.Marker sliceMarker = myBuilder.mark();
sliceMarker.done(PyElementTypes.EMPTY_EXPRESSION);
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
nextToken();
exprStart.done(PyElementTypes.SLICE_EXPRESSION);
return;
} else {
if (atToken(PyTokenTypes.COLON)) {
PsiBuilder.Marker sliceMarker = myBuilder.mark();
sliceMarker.done(PyElementTypes.EMPTY_EXPRESSION);
} else {
parseSingleExpression(false);
}
if (!BRACKET_COLON_COMMA.contains(myBuilder.getTokenType())) {
myBuilder.error(message("PARSE.expected.colon.or.rbracket"));
}
if (matchToken(PyTokenTypes.COLON)) {
parseSingleExpression(false);
}
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
if (!BRACKET_OR_COMMA.contains(myBuilder.getTokenType())) {
myBuilder.error("']' or ',' expected");
}
}
parseSliceListTail(exprStart, null);
}
/**
* Named list operators
*
* @param b PerlBuilder
* @param l Parsing level
* @return parsing result
*/
public static boolean isListOperator(PsiBuilder b, int l) {
PerlTokenData prevTokenData = ((PerlBuilder) b).lookupToken(-1);
if (prevTokenData != null && prevTokenData.getTokenType() == OPERATOR_DEREFERENCE) return false;
IElementType tokenType = b.getTokenType();
IElementType nextTokenType = b.lookAhead(1);
if (CONVERTABLE_TOKENS.contains(tokenType)
&& nextTokenType != LEFT_PAREN // not function call
&& !PACKAGE_TOKENS.contains(nextTokenType) // not method Package::
&& !(nextTokenType == IDENTIFIER
&& ((PerlBuilder) b)
.isKnownPackage(
((PerlBuilder) b).lookupToken(1).getTokenText())) // not Method Package
)
// todo we should check current namespace here
return !PerlSubUtil.BUILT_IN_UNARY.contains(b.getTokenText());
else if (PACKAGE_TOKENS.contains(tokenType)
&& CONVERTABLE_TOKENS.contains(nextTokenType)
&& b.lookAhead(2) != LEFT_PAREN)
return !PerlSubUtil.isUnary(
b.getTokenText(), ((PerlBuilder) b).lookupToken(1).getTokenText());
return false;
}
/**
* Statement recovery function. Should not consume token, only check;
*
* @param b PerlBuilder
* @param l parsing level
* @return parsing result
*/
public static boolean recoverStatement(PsiBuilder b, int l) {
assert b instanceof PerlBuilder;
if (!((PerlBuilder) b).isRecoveringStatement()) ((PerlBuilder) b).startRecovery();
IElementType currentTokenType = b.getTokenType();
// System.err.println("Checking " + b.getTokenText() + currentTokenType);
if (currentTokenType == null // got end of file
|| ((PerlBuilder) b).getBracesLevel() == 0
&& ( // we are not in braced statement
UNCONDITIONAL_STATEMENT_RECOVERY_TOKENS.contains(
currentTokenType) // got semi, package, end of regex, use, compound or suffix
|| currentTokenType == RESERVED_SUB
&& STATEMENT_RECOVERY_SUB_SUFFIX.contains(b.lookAhead(1)) // got sub definition
)) {
((PerlBuilder) b).stopRecovery();
return false;
}
if (currentTokenType == LEFT_BRACE) ((PerlBuilder) b).openBrace();
else if (currentTokenType == RIGHT_BRACE) ((PerlBuilder) b).closeBrace();
return true;
}
boolean typeExtends(IElementType child, IElementType parent) {
if (child == parent) return true;
if (extendsSets != null) {
for (TokenSet set : extendsSets) {
if (set.contains(child) && set.contains(parent)) return true;
}
}
return altExtendsChecker != null && altExtendsChecker.process(child, parent);
}
@Nullable
private PsiBuilder.Marker parseUnary(final PsiBuilder builder) {
final IElementType tokenType = builder.getTokenType();
if (PREFIX_OPS.contains(tokenType)) {
final PsiBuilder.Marker unary = builder.mark();
builder.advanceLexer();
final PsiBuilder.Marker operand = parseUnary(builder);
if (operand == null) {
error(builder, JavaErrorMessages.message("expected.expression"));
}
unary.done(JavaElementType.PREFIX_EXPRESSION);
return unary;
} else if (tokenType == JavaTokenType.LPARENTH) {
final PsiBuilder.Marker typeCast = builder.mark();
builder.advanceLexer();
ReferenceParser.TypeInfo typeInfo =
myParser
.getReferenceParser()
.parseTypeInfo(
builder,
ReferenceParser.EAT_LAST_DOT
| ReferenceParser.WILDCARD
| ReferenceParser.CONJUNCTIONS
| ReferenceParser.INCOMPLETE_ANNO);
if (typeInfo == null || !expect(builder, JavaTokenType.RPARENTH)) {
typeCast.rollbackTo();
return parsePostfix(builder);
}
if (PREF_ARITHMETIC_OPS.contains(builder.getTokenType()) && !typeInfo.isPrimitive) {
typeCast.rollbackTo();
return parsePostfix(builder);
}
final PsiBuilder.Marker expr = parseUnary(builder);
if (expr == null) {
if (!typeInfo
.isParameterized) { // cannot parse correct parenthesized expression after correct
// parameterized type
typeCast.rollbackTo();
return parsePostfix(builder);
} else {
error(builder, JavaErrorMessages.message("expected.expression"));
}
}
typeCast.done(JavaElementType.TYPE_CAST_EXPRESSION);
return typeCast;
} else {
return parsePostfix(builder);
}
}
/**
* There is a possible case that given base node doesn't have {@link
* AlignmentInColumnsConfig#getTargetDeclarationTypes() target type} but its first child node or
* first child node of the first child node etc does.
*
* <p>This method tries to derive node of the target type from the given node.
*
* @param baseNode base node to process
* @param targetTypes target node types
* @return base node or its first descendant child that has {@link
* AlignmentInColumnsConfig#getTargetDeclarationTypes() target type} target type if the one if
* found; <code>null</code> otherwise
*/
@Nullable
private static ASTNode deriveNodeOfTargetType(ASTNode baseNode, TokenSet targetTypes) {
if (targetTypes.contains(baseNode.getElementType())) {
return baseNode;
}
for (ASTNode node = baseNode; node != null; node = node.getFirstChildNode()) {
IElementType nodeType = node.getElementType();
if (targetTypes.contains(nodeType)) {
return node;
}
}
return null;
}
/**
* Tries to calculate given line's indent column assuming that there might be a comment at the
* given indent offset (see {@link #getCommentPrefix(IElementType)}).
*
* @param line target line
* @param indentOffset start indent offset to use for the given line
* @param lineEndOffset given line's end offset
* @param fallbackColumn column to return if it's not possible to apply comment-specific indent
* calculation rules
* @return given line's indent column to use
*/
private int calcIndent(int line, int indentOffset, int lineEndOffset, int fallbackColumn) {
final HighlighterIterator it = myEditor.getHighlighter().createIterator(indentOffset);
IElementType tokenType = it.getTokenType();
Language language = tokenType.getLanguage();
TokenSet comments = myComments.get(language);
if (comments == null) {
ParserDefinition definition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
if (definition != null) {
comments = definition.getCommentTokens();
}
if (comments == null) {
return fallbackColumn;
} else {
myComments.put(language, comments);
}
}
if (comments.contains(tokenType) && indentOffset == it.getStart()) {
String prefix = COMMENT_PREFIXES.get(tokenType);
if (prefix == null) {
prefix = getCommentPrefix(tokenType);
}
if (!NO_COMMENT_INFO_MARKER.equals(prefix)) {
final int indentInsideCommentOffset =
CharArrayUtil.shiftForward(
myChars, indentOffset + prefix.length(), lineEndOffset, " \t");
if (indentInsideCommentOffset < lineEndOffset) {
int indent = myEditor.calcColumnNumber(indentInsideCommentOffset, line);
indentAfterUncomment.put(line, indent - prefix.length());
return indent;
}
}
}
return fallbackColumn;
}
void addNestedChildrenSuffix(
List<Block> list,
@Nullable AlignmentProvider.Aligner aligner,
boolean topLevel,
List<ASTNode> children,
int limit) {
for (int i = 1; i < limit; i++) {
ASTNode childNode = children.get(i);
if (canBeCorrectBlock(childNode)) {
IElementType type = childNode.getElementType();
Indent indent =
topLevel || NESTED.contains(type) || type == mIDENT || TokenSets.DOTS.contains(type)
? Indent.getContinuationWithoutFirstIndent()
: Indent.getNoneIndent();
if (aligner != null && TokenSets.DOTS.contains(type)) {
aligner.append(childNode.getPsi());
}
list.add(
new GroovyBlock(
childNode, indent, myWrap, mySettings, myGroovySettings, myAlignmentProvider));
}
}
}
@Nullable
private static PsiBuilder.Marker parseBinary(
final PsiBuilder builder, final ExprType type, final TokenSet ops) {
PsiBuilder.Marker left = parseExpression(builder, type);
if (left == null) return null;
while (true) {
final IElementType tokenType = getGtTokenType(builder);
if (tokenType == null || !ops.contains(tokenType)) break;
final PsiBuilder.Marker binary = left.precede();
advanceGtToken(builder, tokenType);
final PsiBuilder.Marker right = parseExpression(builder, type);
if (right == null) {
error(builder, JavaErrorMessages.message("expected.expression"));
binary.done(JavaElementType.BINARY_EXPRESSION);
return binary;
}
binary.done(JavaElementType.BINARY_EXPRESSION);
left = binary;
}
return left;
}
private boolean parseSliceListTail(
PsiBuilder.Marker exprStart, @Nullable PsiBuilder.Marker sliceOrTupleStart) {
boolean inSlice = sliceOrTupleStart == null;
while (atToken(PyTokenTypes.COMMA)) {
nextToken();
PsiBuilder.Marker sliceItemStart = myBuilder.mark();
parseTestExpression(false, false);
if (matchToken(PyTokenTypes.COLON)) {
inSlice = true;
parseTestExpression(false, false);
if (matchToken(PyTokenTypes.COLON)) {
parseTestExpression(false, false);
}
}
sliceItemStart.done(PyElementTypes.SLICE_ITEM);
if (!BRACKET_OR_COMMA.contains(myBuilder.getTokenType())) {
myBuilder.error("']' or ',' expected");
break;
}
}
checkMatches(PyTokenTypes.RBRACKET, message("PARSE.expected.rbracket"));
if (inSlice) {
if (sliceOrTupleStart != null) {
sliceOrTupleStart.drop();
}
exprStart.done(PyElementTypes.SLICE_EXPRESSION);
}
return inSlice;
}
@NotNull
private static Indent indentIfNotBrace(@NotNull ASTNode child) {
if (BRACES_TOKEN_SET.contains(child.getElementType())) {
return Indent.getNoneIndent();
} else {
return Indent.getNormalIndent();
}
}
@NotNull
public static PsiBuilder.Marker parseArgumentList(final PsiBuilder builder) {
final PsiBuilder.Marker list = builder.mark();
builder.advanceLexer();
boolean first = true;
while (true) {
final IElementType tokenType = builder.getTokenType();
if (first && (ARGS_LIST_END.contains(tokenType) || builder.eof())) break;
if (!first && !ARGS_LIST_CONTINUE.contains(tokenType)) break;
boolean hasError = false;
if (!first) {
if (builder.getTokenType() == JavaTokenType.COMMA) {
builder.advanceLexer();
} else {
hasError = true;
error(builder, JavaErrorMessages.message("expected.comma.or.rparen"));
emptyExpression(builder);
}
}
first = false;
final PsiBuilder.Marker arg = parse(builder);
if (arg == null) {
if (!hasError) {
error(builder, JavaErrorMessages.message("expected.expression"));
emptyExpression(builder);
}
if (!ARGS_LIST_CONTINUE.contains(builder.getTokenType())) break;
if (builder.getTokenType() != JavaTokenType.COMMA && !builder.eof()) {
builder.advanceLexer();
}
}
}
final boolean closed =
JavaParserUtil.expectOrError(
builder, JavaTokenType.RPARENTH, JavaErrorMessages.message("expected.rparen"));
list.done(JavaElementType.EXPRESSION_LIST);
if (!closed) {
list.setCustomEdgeTokenBinders(null, GREEDY_RIGHT_EDGE_PROCESSOR);
}
return list;
}
public static boolean isPrecededBy(
@Nullable ASTNode node, TokenSet expectedTypes, IElementType... skipTypes) {
ASTNode prevNode = node == null ? null : node.getTreePrev();
while (prevNode != null && (isWhitespaceOrEmpty(prevNode) || isOneOf(prevNode, skipTypes))) {
prevNode = prevNode.getTreePrev();
}
if (prevNode == null) return false;
return expectedTypes.contains(prevNode.getElementType());
}
/**
* parser for print/say/printf filehandle
*
* @param b PerlBuilder
* @param l parsing level
* @return parsing result
*/
public static boolean parsePrintHandle(PsiBuilder b, int l) {
IElementType currentTokenType = b.getTokenType();
IElementType nextTokenType = b.lookAhead(1);
assert b instanceof PerlBuilder;
if (CONVERTABLE_TOKENS.contains(currentTokenType) // it's identifier
&& !PRINT_HANDLE_NEGATE_SUFFIX.contains(nextTokenType) // no negation tokens
&& !PerlSubUtil.BUILT_IN.contains(
b.getTokenText()) // it's not built in. crude, probably we should check any known sub
) {
PsiBuilder.Marker m = b.mark();
b.advanceLexer();
m.collapse(HANDLE);
return true;
}
return false;
}
@Nullable
public static ASTNode skipWhitespaceCommentsAndTokens(final ASTNode node, TokenSet alsoSkip) {
ASTNode element = node;
while (true) {
if (element == null) return null;
if (!isWhitespaceOrComment(element) && !alsoSkip.contains(element.getElementType())) break;
element = element.getTreeNext();
}
return element;
}
@NotNull
private Indent calcIndent(@NotNull ASTNode child) {
IElementType parentType = myNode.getElementType();
if (parentType == TOP_LEVEL) return Indent.getAbsoluteNoneIndent();
if (BLOCKS_TOKEN_SET.contains(parentType)) {
return indentIfNotBrace(child);
} else {
return Indent.getNoneIndent();
}
}
/**
* Parsing label declaration LABEL:
*
* @param b PerlBuilder
* @param l parsing level
* @return parsing result
*/
public static boolean parseLabelDeclaration(PsiBuilder b, int l) {
if (CONVERTABLE_TOKENS.contains(b.getTokenType()) && b.lookAhead(1) == COLON) {
PsiBuilder.Marker m = b.mark();
b.advanceLexer();
m.collapse(LABEL);
b.advanceLexer();
return true;
}
return false;
}
/**
* Checks for version token and convert if necessary
*
* @param b PerlBuilder
* @param l parsing level
* @return parsing result
*/
public static boolean parsePerlVersion(PsiBuilder b, int l) {
if (b.getTokenType() == NUMBER_VERSION) return consumeToken(b, NUMBER_VERSION);
else if (VERSION_TOKENS.contains(b.getTokenType())) {
PsiBuilder.Marker m = b.mark();
b.advanceLexer();
m.collapse(NUMBER_VERSION);
return true;
}
return false;
}
public GoBinaryExpressionBlock(
ASTNode node, Alignment alignment, Wrap wrap, CommonCodeStyleSettings settings) {
super(node, alignment, Indent.getNormalIndent(), wrap, settings);
GoBinaryExpression psi = node.getPsi(GoBinaryExpression.class);
if (psi != null && EMPTY_SPACE_TOKENS.contains(psi.getOperator())) {
spacing = EMPTY_SPACING_KEEP_LINE_BREAKS;
} else {
spacing = BASIC_SPACING_KEEP_LINE_BREAKS;
}
}
@Nullable
protected Indent getChildIndent() {
IElementType parentType =
myNode.getElementType(); // always the parent since isIncomplete is false
if (BLOCKS_TOKEN_SET.contains(parentType)) {
return Indent.getNormalIndent();
} else {
return Indent.getNoneIndent();
}
}
public static boolean convertBracedString(PsiBuilder b, int l) {
if (CONVERTABLE_TOKENS.contains(b.getTokenType()) && b.lookAhead(1) == RIGHT_BRACE) {
// fixme shouldn't we add string_sq here?
PsiBuilder.Marker m = b.mark();
b.advanceLexer();
m.collapse(STRING_CONTENT);
return true;
}
return false;
}
// @todo this is really raw
public static boolean parseVariableAttributes(PsiBuilder b, int l) {
PsiBuilder.Marker m = null;
while (!b.eof() && !VARIABLE_ATTRIBUTE_STOP_TOKENS.contains(b.getTokenType())) {
if (m == null) m = b.mark();
b.advanceLexer();
}
if (m != null) m.collapse(VAR_ATTRIBUTE);
return true;
}
private static List<IElementType> findSubNodeTypes(ASTNode node, TokenSet types) {
List<IElementType> foundTypes = new SmartList<IElementType>();
for (ASTNode child = node.getFirstChildNode();
child != null && child.getTreeParent() == node;
child = child.getTreeNext()) {
IElementType type = child.getElementType();
if (types.contains(type)) {
foundTypes.add(type);
}
}
return foundTypes;
}
/**
* Checks and parses bareword filehandle for <FH> operations
*
* @param b PerlBuilder
* @param l parsing level
* @return parsing result
*/
public static boolean parseReadHandle(PsiBuilder b, int l) {
IElementType currentTokenType = b.getTokenType();
IElementType nextTokenType = b.lookAhead(1);
if (CONVERTABLE_TOKENS.contains(currentTokenType) && nextTokenType == OPERATOR_GT_NUMERIC) {
PsiBuilder.Marker m = b.mark();
b.advanceLexer();
m.collapse(HANDLE);
return true;
}
return false;
}
@Nullable
private PsiBuilder.Marker parseBinary(
final PsiBuilder builder, final ExprType type, final TokenSet ops) {
PsiBuilder.Marker result = parseExpression(builder, type);
if (result == null) return null;
int operandCount = 1;
IElementType tokenType = getGtTokenType(builder);
IElementType currentExprTokenType = tokenType;
while (true) {
if (tokenType == null || !ops.contains(tokenType)) break;
advanceGtToken(builder, tokenType);
final PsiBuilder.Marker right = parseExpression(builder, type);
operandCount++;
tokenType = getGtTokenType(builder);
if (tokenType == null
|| !ops.contains(tokenType)
|| tokenType != currentExprTokenType
|| right == null) {
// save
result = result.precede();
if (right == null) {
error(builder, JavaErrorMessages.message("expected.expression"));
}
result.done(
operandCount > 2
? JavaElementType.POLYADIC_EXPRESSION
: JavaElementType.BINARY_EXPRESSION);
if (right == null) break;
currentExprTokenType = tokenType;
operandCount = 1;
}
}
return result;
}
/**
* Merges sequence [package] identifier to a package
*
* @param b PerlBuilder
* @param l parsing level
* @return result
*/
public static boolean mergePackageName(PsiBuilder b, int l) {
IElementType tokenType = b.getTokenType();
if (tokenType == PACKAGE) {
b.advanceLexer();
return true;
} else if (PACKAGE_TOKENS.contains(tokenType) && CONVERTABLE_TOKENS.contains(b.lookAhead(1))) {
PsiBuilder.Marker m = b.mark();
b.advanceLexer();
b.advanceLexer();
m.collapse(PACKAGE);
return true;
} else if (PACKAGE_TOKENS.contains(tokenType) // explicit package name, like Foo::->method()
|| CONVERTABLE_TOKENS.contains(tokenType) // single word package
) {
PsiBuilder.Marker m = b.mark();
b.advanceLexer();
m.collapse(PACKAGE);
return true;
}
return false;
}
@Override
@NotNull
public Map<TodoIndexEntry, Integer> map(@NotNull final FileContent inputData) {
if (IndexPatternUtil.getIndexPatternCount() > 0) {
final CharSequence chars = inputData.getContentAsText();
final OccurrenceConsumer occurrenceConsumer = new OccurrenceConsumer(null, true);
EditorHighlighter highlighter;
final EditorHighlighter editorHighlighter = inputData.getUserData(EDITOR_HIGHLIGHTER);
if (editorHighlighter != null
&& checkCanUseCachedEditorHighlighter(chars, editorHighlighter)) {
highlighter = editorHighlighter;
} else {
highlighter = HighlighterFactory.createHighlighter(inputData.getProject(), myFile);
highlighter.setText(chars);
}
final int documentLength = chars.length();
BaseFilterLexer.TodoScanningState todoScanningState = null;
final HighlighterIterator iterator = highlighter.createIterator(0);
while (!iterator.atEnd()) {
final IElementType token = iterator.getTokenType();
if (myCommentTokens.contains(token) || CacheUtil.isInComments(token)) {
int start = iterator.getStart();
if (start >= documentLength) break;
int end = iterator.getEnd();
todoScanningState =
BaseFilterLexer.advanceTodoItemsCount(
chars.subSequence(start, Math.min(end, documentLength)),
occurrenceConsumer,
todoScanningState);
if (end > documentLength) break;
}
iterator.advance();
}
final Map<TodoIndexEntry, Integer> map = new HashMap<>();
for (IndexPattern pattern : IndexPatternUtil.getIndexPatterns()) {
final int count = occurrenceConsumer.getOccurrenceCount(pattern);
if (count > 0) {
map.put(
new TodoIndexEntry(pattern.getPatternString(), pattern.isCaseSensitive()), count);
}
}
return map;
}
return Collections.emptyMap();
}
/**
* Replaces identifier as a variable name
*
* @param b Perlbuilder
* @param l parsing level
* @return parsing result
*/
public static boolean convertIdentifier(PsiBuilder b, int l, IElementType tokenType) {
IElementType currentTokenType = b.getTokenType();
if (currentTokenType == tokenType) {
b.advanceLexer();
return true;
} else if (CONVERTABLE_TOKENS.contains(currentTokenType)) {
PsiBuilder.Marker m = b.mark();
b.advanceLexer();
m.collapse(tokenType);
return true;
}
return false;
}
/**
* Completes namespace, invoked when we are 100% sure that PACKAGE_IDENTIFIER is a package
*
* @param b Perlbuilder
* @param l parsing level
* @return parsing result
*/
public static boolean convertPackageIdentifier(PsiBuilder b, int l) {
IElementType currentTokenType = b.getTokenType();
if (currentTokenType == PACKAGE) {
b.advanceLexer();
return true;
} else if (PACKAGE_TOKENS.contains(currentTokenType)) {
PsiBuilder.Marker m = b.mark();
b.advanceLexer();
m.collapse(PACKAGE);
return true;
}
return false;
}
@Nullable
private PsiBuilder.Marker parsePostfix(final PsiBuilder builder) {
PsiBuilder.Marker operand = parsePrimary(builder, null, -1);
if (operand == null) return null;
while (POSTFIX_OPS.contains(builder.getTokenType())) {
final PsiBuilder.Marker postfix = operand.precede();
builder.advanceLexer();
postfix.done(JavaElementType.POSTFIX_EXPRESSION);
operand = postfix;
}
return operand;
}