private static void doTest( @NonNls final String text, final Parser parser, @NonNls final String expected) { final PsiBuilder builder = createBuilder(text); final PsiBuilder.Marker rootMarker = builder.mark(); parser.parse(builder); rootMarker.done(ROOT); // check light tree composition final FlyweightCapableTreeStructure<LighterASTNode> lightTree = builder.getLightTree(); assertEquals(expected, DebugUtil.lightTreeToString(lightTree, false)); // verify that light tree can be taken multiple times final FlyweightCapableTreeStructure<LighterASTNode> lightTree2 = builder.getLightTree(); assertEquals(expected, DebugUtil.lightTreeToString(lightTree2, false)); // check heavy tree composition final ASTNode root = builder.getTreeBuilt(); assertEquals(expected, DebugUtil.nodeTreeToString(root, false)); // check heavy vs. light tree merging final PsiBuilder builder2 = createBuilder(text); final PsiBuilder.Marker rootMarker2 = builder2.mark(); parser.parse(builder2); rootMarker2.done(ROOT); DiffTree.diff( new ASTStructure(root), builder2.getLightTree(), new ShallowNodeComparator<ASTNode, LighterASTNode>() { @Override public ThreeState deepEqual(ASTNode oldNode, LighterASTNode newNode) { return ThreeState.UNSURE; } @Override public boolean typesEqual(ASTNode oldNode, LighterASTNode newNode) { return true; } @Override public boolean hashCodesEqual(ASTNode oldNode, LighterASTNode newNode) { return true; } }, new DiffTreeChangeBuilder<ASTNode, LighterASTNode>() { @Override public void nodeReplaced(@NotNull ASTNode oldChild, @NotNull LighterASTNode newChild) { fail("replaced(" + oldChild + "," + newChild.getTokenType() + ")"); } @Override public void nodeDeleted(@NotNull ASTNode oldParent, @NotNull ASTNode oldNode) { fail("deleted(" + oldParent + "," + oldNode + ")"); } @Override public void nodeInserted( @NotNull ASTNode oldParent, @NotNull LighterASTNode newNode, int pos) { fail("inserted(" + oldParent + "," + newNode.getTokenType() + ")"); } }); }
public void parse(PsiBuilder builder) { final PsiBuilder.Marker root = builder.mark(); PsiBuilder.Marker error = null; while (!builder.eof()) { final String token = builder.getTokenText(); if ("?".equals(token)) error = builder.mark(); builder.advanceLexer(); if (error != null) { error.error("test error 2"); error = null; } } root.done(this); }
public void parse(PsiBuilder builder) { final PsiBuilder.Marker root = builder.mark(); PsiBuilder.Marker nested = null; while (!builder.eof()) { final String token = builder.getTokenText(); if ("[".equals(token) && nested == null) { nested = builder.mark(); } builder.advanceLexer(); if ("]".equals(token) && nested != null) { nested.collapse(myCHAMELEON_2); nested.precede().done(OTHER); nested = null; builder.error("test error 1"); } } if (nested != null) nested.drop(); root.done(this); }
public static boolean parseTokens( PsiBuilder builder, boolean smart, int pin, IElementType... tokens) { PsiBuilder.Marker marker = builder.mark(); boolean result = consumeTokens(builder, smart, pin, tokens); if (!result) { marker.rollbackTo(); } else { marker.drop(); } return result; }
private static boolean reportError( PsiBuilder builder, ErrorState state, Frame frame, IElementType elementType, boolean force, boolean advance) { String expectedText = state.getExpectedText(builder); boolean notEmpty = StringUtil.isNotEmpty(expectedText); if (force || notEmpty || advance) { String gotText = builder.eof() ? "unexpected end of file" : notEmpty ? "got '" + builder.getTokenText() + "'" : "'" + builder.getTokenText() + "' unexpected"; String message = expectedText + gotText; if (advance) { PsiBuilder.Marker mark = builder.mark(); builder.advanceLexer(); mark.error(message); } else if (!force) { PsiBuilder.Marker extensionMarker = null; IElementType extensionTokenType = null; PsiBuilderImpl.ProductionMarker latestDoneMarker = elementType == null ? null : (PsiBuilderImpl.ProductionMarker) builder.getLatestDoneMarker(); if (latestDoneMarker != null && frame.position >= latestDoneMarker.getStartIndex() && frame.position <= latestDoneMarker.getEndIndex()) { extensionMarker = ((PsiBuilder.Marker) latestDoneMarker).precede(); extensionTokenType = latestDoneMarker.getTokenType(); ((PsiBuilder.Marker) latestDoneMarker).drop(); } builder.error(message); if (extensionMarker != null) extensionMarker.done(extensionTokenType); } else { builder.error(message); } builder.eof(); // skip whitespaces frame.errorReportedAt = builder.rawTokenIndex(); return true; } return false; }
// complex enter/exit methods pair with frame object public static PsiBuilder.Marker enter_section_( PsiBuilder builder, int level, int modifiers, @Nullable String frameName) { PsiBuilder.Marker marker = builder.mark(); enter_section_impl_(builder, level, modifiers, frameName); return marker; }
// simple enter/exit methods pair that doesn't require frame object public static PsiBuilder.Marker enter_section_(PsiBuilder builder) { return builder.mark(); }
public static boolean parseAsTree( ErrorState state, final PsiBuilder builder, int level, final IElementType chunkType, boolean checkBraces, final Parser parser, final Parser eatMoreCondition) { final LinkedList<Pair<PsiBuilder.Marker, PsiBuilder.Marker>> parenList = new LinkedList<Pair<PsiBuilder.Marker, PsiBuilder.Marker>>(); final LinkedList<Pair<PsiBuilder.Marker, Integer>> siblingList = new LinkedList<Pair<PsiBuilder.Marker, Integer>>(); PsiBuilder.Marker marker = null; final Runnable checkSiblingsRunnable = new Runnable() { @Override public void run() { main: while (!siblingList.isEmpty()) { final Pair<PsiBuilder.Marker, PsiBuilder.Marker> parenPair = parenList.peek(); final int rating = siblingList.getFirst().second; int count = 0; for (Pair<PsiBuilder.Marker, Integer> pair : siblingList) { if (pair.second != rating || parenPair != null && pair.first == parenPair.second) break main; if (++count >= MAX_CHILDREN_IN_TREE) { PsiBuilder.Marker parentMarker = pair.first.precede(); parentMarker.setCustomEdgeTokenBinders( WhitespacesBinders.GREEDY_LEFT_BINDER, null); while (count-- > 0) { siblingList.removeFirst(); } parentMarker.done(chunkType); siblingList.addFirst(Pair.create(parentMarker, rating + 1)); continue main; } } break; } } }; boolean checkParens = state.braces != null && checkBraces; int totalCount = 0; int tokenCount = 0; if (checkParens) { int tokenIdx = -1; while (builder.rawLookup(tokenIdx) == TokenType.WHITE_SPACE) tokenIdx--; LighterASTNode doneMarker = builder.rawLookup(tokenIdx) == state.braces[0].getLeftBraceType() ? builder.getLatestDoneMarker() : null; if (doneMarker != null && doneMarker.getStartOffset() == builder.rawTokenTypeStart(tokenIdx) && doneMarker.getTokenType() == TokenType.ERROR_ELEMENT) { parenList.add( Pair.create(((PsiBuilder.Marker) doneMarker).precede(), (PsiBuilder.Marker) null)); } } while (true) { final IElementType tokenType = builder.getTokenType(); if (checkParens && (tokenType == state.braces[0].getLeftBraceType() || tokenType == state.braces[0].getRightBraceType() && !parenList.isEmpty())) { if (marker != null) { marker.done(chunkType); siblingList.addFirst(Pair.create(marker, 1)); marker = null; tokenCount = 0; } if (tokenType == state.braces[0].getLeftBraceType()) { final Pair<PsiBuilder.Marker, Integer> prev = siblingList.peek(); parenList.addFirst(Pair.create(builder.mark(), prev == null ? null : prev.first)); } checkSiblingsRunnable.run(); builder.advanceLexer(); if (tokenType == state.braces[0].getRightBraceType()) { final Pair<PsiBuilder.Marker, PsiBuilder.Marker> pair = parenList.removeFirst(); pair.first.done(chunkType); // drop all markers inside parens while (!siblingList.isEmpty() && siblingList.getFirst().first != pair.second) { siblingList.removeFirst(); } siblingList.addFirst(Pair.create(pair.first, 1)); checkSiblingsRunnable.run(); } } else { if (marker == null) { marker = builder.mark(); marker.setCustomEdgeTokenBinders(WhitespacesBinders.GREEDY_LEFT_BINDER, null); } final boolean result = (!parenList.isEmpty() || eatMoreCondition.parse(builder, level + 1)) && parser.parse(builder, level + 1); if (result) { tokenCount++; totalCount++; } if (!result) { break; } } if (tokenCount >= MAX_CHILDREN_IN_TREE && marker != null) { marker.done(chunkType); siblingList.addFirst(Pair.create(marker, 1)); checkSiblingsRunnable.run(); marker = null; tokenCount = 0; } } if (marker != null) { marker.drop(); } for (Pair<PsiBuilder.Marker, PsiBuilder.Marker> pair : parenList) { pair.first.drop(); } return totalCount != 0; }
public static PsiBuilder.Marker enter_section_( PsiBuilder builder, int level, int modifiers, IElementType elementType, String frameName) { PsiBuilder.Marker marker = builder.mark(); enter_section_impl_(builder, level, modifiers, elementType, frameName); return marker; }
private void doParse(final PsiBuilder builder) { final PsiBuilder.Marker root = builder.mark(); FileParser.parse(builder); root.done(this); }
public static boolean exitErrorRecordingSection( PsiBuilder builder_, int level, boolean result, boolean pinned, @NotNull String sectionType, @Nullable Parser eatMore) { ErrorState state = ErrorState.get(builder_); Frame frame = state.levelCheck.pollLast(); int initialOffset = builder_.getCurrentOffset(); if (frame == null || level != frame.level || !sectionType.equals(frame.section)) { LOG.error( "Unbalanced error section: got " + new Frame().init(initialOffset, level, sectionType, "", 0) + ", expected " + frame); if (frame != null) state.FRAMES.recycle(frame); return result; } if (sectionType == _SECTION_AND_ || sectionType == _SECTION_NOT_) { state.predicateCount--; if (sectionType == _SECTION_NOT_) state.predicateSign = !state.predicateSign; state.FRAMES.recycle(frame); return result; } if (!result && !pinned && initialOffset == frame.offset && state.lastExpectedVariantOffset == frame.offset && frame.name != null && state.variants.size() - frame.variantCount > 1) { state.clearVariants(true, frame.variantCount); addVariantInner(state, initialOffset, frame.name); } if (sectionType == _SECTION_RECOVER_ && !state.suppressErrors && eatMore != null) { state.suppressErrors = true; final boolean eatMoreFlagOnce = !builder_.eof() && eatMore.parse(builder_, frame.level + 1); final int lastErrorPos = getLastVariantOffset(state, initialOffset); boolean eatMoreFlag = eatMoreFlagOnce || frame.offset == initialOffset && lastErrorPos > frame.offset; final LighterASTNode latestDoneMarker = (pinned || result) && (state.altMode || lastErrorPos > initialOffset) && eatMoreFlagOnce ? builder_.getLatestDoneMarker() : null; PsiBuilder.Marker extensionMarker = null; IElementType extensionTokenType = null; if (latestDoneMarker instanceof PsiBuilder.Marker) { extensionMarker = ((PsiBuilder.Marker) latestDoneMarker).precede(); extensionTokenType = latestDoneMarker.getTokenType(); ((PsiBuilder.Marker) latestDoneMarker).drop(); } // advance to the last error pos // skip tokens until lastErrorPos. parseAsTree might look better here... int parenCount = 0; while (eatMoreFlag && builder_.getCurrentOffset() < lastErrorPos) { if (state.braces != null) { if (builder_.getTokenType() == state.braces[0].getLeftBraceType()) parenCount++; else if (builder_.getTokenType() == state.braces[0].getRightBraceType()) parenCount--; } builder_.advanceLexer(); eatMoreFlag = parenCount != 0 || eatMore.parse(builder_, frame.level + 1); } boolean errorReported = frame.errorReportedAt == initialOffset; if (errorReported) { if (eatMoreFlag) { builder_.advanceLexer(); parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); } } else if (eatMoreFlag) { String tokenText = builder_.getTokenText(); String expectedText = state.getExpectedText(builder_); PsiBuilder.Marker mark = builder_.mark(); builder_.advanceLexer(); final String gotText = !expectedText.isEmpty() ? "got '" + tokenText + "'" : "'" + tokenText + "' unexpected"; mark.error(expectedText + gotText); parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); errorReported = true; } else if (eatMoreFlagOnce || (!result && frame.offset != builder_.getCurrentOffset())) { reportError(state, builder_, true); errorReported = true; } if (extensionMarker != null) { extensionMarker.done(extensionTokenType); } state.suppressErrors = false; if (errorReported || result) { state.clearVariants(true, 0); state.clearVariants(false, 0); state.lastExpectedVariantOffset = -1; } if (!result && eatMoreFlagOnce && frame.offset != builder_.getCurrentOffset()) result = true; } else if (!result && pinned && frame.errorReportedAt < 0) { // do not report if there're errors after current offset if (getLastVariantOffset(state, initialOffset) == initialOffset) { // do not force, inner recoverRoot might have skipped some tokens if (reportError(state, builder_, false)) { frame.errorReportedAt = initialOffset; } } } // propagate errorReportedAt up the stack to avoid duplicate reporting Frame prevFrame = state.levelCheck.isEmpty() ? null : state.levelCheck.getLast(); if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt) prevFrame.errorReportedAt = frame.errorReportedAt; state.FRAMES.recycle(frame); return result; }