private static void doTest(
@NonNls final String text, final Parser parser, @NonNls final String expected) {
final PsiBuilder builder = createBuilder(text);
final PsiBuilder.Marker rootMarker = builder.mark();
parser.parse(builder);
rootMarker.done(ROOT);
// check light tree composition
final FlyweightCapableTreeStructure<LighterASTNode> lightTree = builder.getLightTree();
assertEquals(expected, DebugUtil.lightTreeToString(lightTree, false));
// verify that light tree can be taken multiple times
final FlyweightCapableTreeStructure<LighterASTNode> lightTree2 = builder.getLightTree();
assertEquals(expected, DebugUtil.lightTreeToString(lightTree2, false));
// check heavy tree composition
final ASTNode root = builder.getTreeBuilt();
assertEquals(expected, DebugUtil.nodeTreeToString(root, false));
// check heavy vs. light tree merging
final PsiBuilder builder2 = createBuilder(text);
final PsiBuilder.Marker rootMarker2 = builder2.mark();
parser.parse(builder2);
rootMarker2.done(ROOT);
DiffTree.diff(
new ASTStructure(root),
builder2.getLightTree(),
new ShallowNodeComparator<ASTNode, LighterASTNode>() {
@Override
public ThreeState deepEqual(ASTNode oldNode, LighterASTNode newNode) {
return ThreeState.UNSURE;
}
@Override
public boolean typesEqual(ASTNode oldNode, LighterASTNode newNode) {
return true;
}
@Override
public boolean hashCodesEqual(ASTNode oldNode, LighterASTNode newNode) {
return true;
}
},
new DiffTreeChangeBuilder<ASTNode, LighterASTNode>() {
@Override
public void nodeReplaced(@NotNull ASTNode oldChild, @NotNull LighterASTNode newChild) {
fail("replaced(" + oldChild + "," + newChild.getTokenType() + ")");
}
@Override
public void nodeDeleted(@NotNull ASTNode oldParent, @NotNull ASTNode oldNode) {
fail("deleted(" + oldParent + "," + oldNode + ")");
}
@Override
public void nodeInserted(
@NotNull ASTNode oldParent, @NotNull LighterASTNode newNode, int pos) {
fail("inserted(" + oldParent + "," + newNode.getTokenType() + ")");
}
});
}
public static boolean parseTokens(
PsiBuilder builder, boolean smart, int pin, IElementType... tokens) {
PsiBuilder.Marker marker = builder.mark();
boolean result = consumeTokens(builder, smart, pin, tokens);
if (!result) {
marker.rollbackTo();
} else {
marker.drop();
}
return result;
}
public void parse(PsiBuilder builder) {
final PsiBuilder.Marker root = builder.mark();
PsiBuilder.Marker error = null;
while (!builder.eof()) {
final String token = builder.getTokenText();
if ("?".equals(token)) error = builder.mark();
builder.advanceLexer();
if (error != null) {
error.error("test error 2");
error = null;
}
}
root.done(this);
}
private static boolean reportError(
PsiBuilder builder,
ErrorState state,
Frame frame,
IElementType elementType,
boolean force,
boolean advance) {
String expectedText = state.getExpectedText(builder);
boolean notEmpty = StringUtil.isNotEmpty(expectedText);
if (force || notEmpty || advance) {
String gotText =
builder.eof()
? "unexpected end of file"
: notEmpty
? "got '" + builder.getTokenText() + "'"
: "'" + builder.getTokenText() + "' unexpected";
String message = expectedText + gotText;
if (advance) {
PsiBuilder.Marker mark = builder.mark();
builder.advanceLexer();
mark.error(message);
} else if (!force) {
PsiBuilder.Marker extensionMarker = null;
IElementType extensionTokenType = null;
PsiBuilderImpl.ProductionMarker latestDoneMarker =
elementType == null
? null
: (PsiBuilderImpl.ProductionMarker) builder.getLatestDoneMarker();
if (latestDoneMarker != null
&& frame.position >= latestDoneMarker.getStartIndex()
&& frame.position <= latestDoneMarker.getEndIndex()) {
extensionMarker = ((PsiBuilder.Marker) latestDoneMarker).precede();
extensionTokenType = latestDoneMarker.getTokenType();
((PsiBuilder.Marker) latestDoneMarker).drop();
}
builder.error(message);
if (extensionMarker != null) extensionMarker.done(extensionTokenType);
} else {
builder.error(message);
}
builder.eof(); // skip whitespaces
frame.errorReportedAt = builder.rawTokenIndex();
return true;
}
return false;
}
private static void close_marker_impl_(
Frame frame, PsiBuilder.Marker marker, IElementType elementType, boolean result) {
if (marker == null) return;
if (result) {
if (elementType != null) {
marker.done(elementType);
} else {
marker.drop();
}
} else {
if (frame != null) {
int position = ((PsiBuilderImpl.ProductionMarker) marker).getStartIndex();
if (frame.errorReportedAt > position) {
frame.errorReportedAt = frame.errorReportedAtPrev;
}
}
marker.rollbackTo();
}
}
private static void close_frame_impl_(
ErrorState state,
Frame frame,
PsiBuilder builder,
PsiBuilder.Marker marker,
IElementType elementType,
boolean result,
boolean pinned) {
if (elementType != null && marker != null) {
if ((frame.modifiers & _COLLAPSE_) != 0) {
PsiBuilderImpl.ProductionMarker last =
result || pinned
? (PsiBuilderImpl.ProductionMarker) builder.getLatestDoneMarker()
: null;
if (last != null
&& last.getStartIndex() == frame.position
&& state.typeExtends(last.getTokenType(), elementType)) {
IElementType resultType = last.getTokenType();
((PsiBuilder.Marker) last).drop();
marker.done(resultType);
return;
}
}
if (result || pinned) {
if ((frame.modifiers & _UPPER_) != 0) {
marker.drop();
frame.parentFrame.elementType = elementType;
} else if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
marker.done(elementType);
frame.leftMarker.precede().done(((LighterASTNode) frame.leftMarker).getTokenType());
frame.leftMarker.drop();
} else if ((frame.modifiers & _LEFT_) != 0 && frame.leftMarker != null) {
marker.drop();
frame.leftMarker.precede().done(elementType);
} else {
if (frame.level == 0) builder.eof(); // skip whitespaces
marker.done(elementType);
}
} else {
close_marker_impl_(frame, marker, null, false);
}
} else if (result || pinned) {
if (marker != null) marker.drop();
if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
frame.leftMarker.precede().done(((LighterASTNode) frame.leftMarker).getTokenType());
frame.leftMarker.drop();
}
} else {
close_marker_impl_(frame, marker, null, false);
}
}
public void parse(PsiBuilder builder) {
final PsiBuilder.Marker root = builder.mark();
PsiBuilder.Marker nested = null;
while (!builder.eof()) {
final String token = builder.getTokenText();
if ("[".equals(token) && nested == null) {
nested = builder.mark();
}
builder.advanceLexer();
if ("]".equals(token) && nested != null) {
nested.collapse(myCHAMELEON_2);
nested.precede().done(OTHER);
nested = null;
builder.error("test error 1");
}
}
if (nested != null) nested.drop();
root.done(this);
}
private static void exit_section_impl_(
ErrorState state,
Frame frame,
PsiBuilder builder,
@Nullable IElementType elementType,
boolean result,
boolean pinned,
@Nullable Parser eatMore) {
int initialPos = builder.rawTokenIndex();
boolean willFail = !result && !pinned;
if (willFail
&& initialPos == frame.position
&& state.lastExpectedVariantPos == frame.position
&& frame.name != null
&& state.variants.size() - frame.variantCount > 1) {
state.clearVariants(true, frame.variantCount);
addVariantInner(state, initialPos, frame.name);
}
int lastErrorPos = getLastVariantPos(state, initialPos);
if (!state.suppressErrors && eatMore != null) {
state.suppressErrors = true;
final boolean eatMoreFlagOnce = !builder.eof() && eatMore.parse(builder, frame.level + 1);
boolean eatMoreFlag =
eatMoreFlagOnce
|| !result && frame.position == initialPos && lastErrorPos > frame.position;
PsiBuilderImpl.ProductionMarker latestDoneMarker =
(pinned || result) && (state.altMode || elementType != null) && eatMoreFlagOnce
? (PsiBuilderImpl.ProductionMarker) builder.getLatestDoneMarker()
: null;
PsiBuilder.Marker extensionMarker = null;
IElementType extensionTokenType = null;
// whitespace prefix makes the very first frame offset bigger than marker start offset which
// is always 0
if (latestDoneMarker != null
&& frame.position >= latestDoneMarker.getStartIndex()
&& frame.position <= latestDoneMarker.getEndIndex()) {
extensionMarker = ((PsiBuilder.Marker) latestDoneMarker).precede();
extensionTokenType = latestDoneMarker.getTokenType();
((PsiBuilder.Marker) latestDoneMarker).drop();
}
// advance to the last error pos
// skip tokens until lastErrorPos. parseAsTree might look better here...
int parenCount = 0;
while ((eatMoreFlag || parenCount > 0) && builder.rawTokenIndex() < lastErrorPos) {
if (state.braces != null) {
if (builder.getTokenType() == state.braces[0].getLeftBraceType()) parenCount++;
else if (builder.getTokenType() == state.braces[0].getRightBraceType()) parenCount--;
}
builder.advanceLexer();
eatMoreFlag = eatMore.parse(builder, frame.level + 1);
}
boolean errorReported =
frame.errorReportedAt == initialPos || !result && frame.errorReportedAt >= frame.position;
if (errorReported) {
if (eatMoreFlag) {
builder.advanceLexer();
parseAsTree(state, builder, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore);
}
} else if (eatMoreFlag) {
errorReported = reportError(builder, state, frame, null, true, true);
parseAsTree(state, builder, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore);
} else if (eatMoreFlagOnce
|| (!result && frame.position != builder.rawTokenIndex())
|| frame.errorReportedAt > initialPos) {
errorReported = reportError(builder, state, frame, null, true, false);
}
if (extensionMarker != null) {
extensionMarker.done(extensionTokenType);
}
state.suppressErrors = false;
if (errorReported || result) {
state.clearVariants(true, 0);
state.clearVariants(false, 0);
state.lastExpectedVariantPos = -1;
}
} else if (!result && pinned && frame.errorReportedAt < 0) {
// do not report if there are errors beyond current position
if (lastErrorPos == initialPos) {
// do not force, inner recoverRoot might have skipped some tokens
reportError(builder, state, frame, elementType, false, false);
} else if (lastErrorPos > initialPos) {
// set error pos here as if it is reported for future reference
frame.errorReportedAt = lastErrorPos;
}
}
// propagate errorReportedAt up the stack to avoid duplicate reporting
Frame prevFrame = willFail && eatMore == null ? null : state.frameStack.peekLast();
if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt) {
prevFrame.errorReportedAt = frame.errorReportedAt;
}
}
public static boolean parseAsTree(
ErrorState state,
final PsiBuilder builder,
int level,
final IElementType chunkType,
boolean checkBraces,
final Parser parser,
final Parser eatMoreCondition) {
final LinkedList<Pair<PsiBuilder.Marker, PsiBuilder.Marker>> parenList =
new LinkedList<Pair<PsiBuilder.Marker, PsiBuilder.Marker>>();
final LinkedList<Pair<PsiBuilder.Marker, Integer>> siblingList =
new LinkedList<Pair<PsiBuilder.Marker, Integer>>();
PsiBuilder.Marker marker = null;
final Runnable checkSiblingsRunnable =
new Runnable() {
@Override
public void run() {
main:
while (!siblingList.isEmpty()) {
final Pair<PsiBuilder.Marker, PsiBuilder.Marker> parenPair = parenList.peek();
final int rating = siblingList.getFirst().second;
int count = 0;
for (Pair<PsiBuilder.Marker, Integer> pair : siblingList) {
if (pair.second != rating || parenPair != null && pair.first == parenPair.second)
break main;
if (++count >= MAX_CHILDREN_IN_TREE) {
PsiBuilder.Marker parentMarker = pair.first.precede();
parentMarker.setCustomEdgeTokenBinders(
WhitespacesBinders.GREEDY_LEFT_BINDER, null);
while (count-- > 0) {
siblingList.removeFirst();
}
parentMarker.done(chunkType);
siblingList.addFirst(Pair.create(parentMarker, rating + 1));
continue main;
}
}
break;
}
}
};
boolean checkParens = state.braces != null && checkBraces;
int totalCount = 0;
int tokenCount = 0;
if (checkParens) {
int tokenIdx = -1;
while (builder.rawLookup(tokenIdx) == TokenType.WHITE_SPACE) tokenIdx--;
LighterASTNode doneMarker =
builder.rawLookup(tokenIdx) == state.braces[0].getLeftBraceType()
? builder.getLatestDoneMarker()
: null;
if (doneMarker != null
&& doneMarker.getStartOffset() == builder.rawTokenTypeStart(tokenIdx)
&& doneMarker.getTokenType() == TokenType.ERROR_ELEMENT) {
parenList.add(
Pair.create(((PsiBuilder.Marker) doneMarker).precede(), (PsiBuilder.Marker) null));
}
}
while (true) {
final IElementType tokenType = builder.getTokenType();
if (checkParens
&& (tokenType == state.braces[0].getLeftBraceType()
|| tokenType == state.braces[0].getRightBraceType() && !parenList.isEmpty())) {
if (marker != null) {
marker.done(chunkType);
siblingList.addFirst(Pair.create(marker, 1));
marker = null;
tokenCount = 0;
}
if (tokenType == state.braces[0].getLeftBraceType()) {
final Pair<PsiBuilder.Marker, Integer> prev = siblingList.peek();
parenList.addFirst(Pair.create(builder.mark(), prev == null ? null : prev.first));
}
checkSiblingsRunnable.run();
builder.advanceLexer();
if (tokenType == state.braces[0].getRightBraceType()) {
final Pair<PsiBuilder.Marker, PsiBuilder.Marker> pair = parenList.removeFirst();
pair.first.done(chunkType);
// drop all markers inside parens
while (!siblingList.isEmpty() && siblingList.getFirst().first != pair.second) {
siblingList.removeFirst();
}
siblingList.addFirst(Pair.create(pair.first, 1));
checkSiblingsRunnable.run();
}
} else {
if (marker == null) {
marker = builder.mark();
marker.setCustomEdgeTokenBinders(WhitespacesBinders.GREEDY_LEFT_BINDER, null);
}
final boolean result =
(!parenList.isEmpty() || eatMoreCondition.parse(builder, level + 1))
&& parser.parse(builder, level + 1);
if (result) {
tokenCount++;
totalCount++;
}
if (!result) {
break;
}
}
if (tokenCount >= MAX_CHILDREN_IN_TREE && marker != null) {
marker.done(chunkType);
siblingList.addFirst(Pair.create(marker, 1));
checkSiblingsRunnable.run();
marker = null;
tokenCount = 0;
}
}
if (marker != null) {
marker.drop();
}
for (Pair<PsiBuilder.Marker, PsiBuilder.Marker> pair : parenList) {
pair.first.drop();
}
return totalCount != 0;
}
private void doParse(final PsiBuilder builder) {
final PsiBuilder.Marker root = builder.mark();
FileParser.parse(builder);
root.done(this);
}
public static boolean exitErrorRecordingSection(
PsiBuilder builder_,
int level,
boolean result,
boolean pinned,
@NotNull String sectionType,
@Nullable Parser eatMore) {
ErrorState state = ErrorState.get(builder_);
Frame frame = state.levelCheck.pollLast();
int initialOffset = builder_.getCurrentOffset();
if (frame == null || level != frame.level || !sectionType.equals(frame.section)) {
LOG.error(
"Unbalanced error section: got "
+ new Frame().init(initialOffset, level, sectionType, "", 0)
+ ", expected "
+ frame);
if (frame != null) state.FRAMES.recycle(frame);
return result;
}
if (sectionType == _SECTION_AND_ || sectionType == _SECTION_NOT_) {
state.predicateCount--;
if (sectionType == _SECTION_NOT_) state.predicateSign = !state.predicateSign;
state.FRAMES.recycle(frame);
return result;
}
if (!result
&& !pinned
&& initialOffset == frame.offset
&& state.lastExpectedVariantOffset == frame.offset
&& frame.name != null
&& state.variants.size() - frame.variantCount > 1) {
state.clearVariants(true, frame.variantCount);
addVariantInner(state, initialOffset, frame.name);
}
if (sectionType == _SECTION_RECOVER_ && !state.suppressErrors && eatMore != null) {
state.suppressErrors = true;
final boolean eatMoreFlagOnce = !builder_.eof() && eatMore.parse(builder_, frame.level + 1);
final int lastErrorPos = getLastVariantOffset(state, initialOffset);
boolean eatMoreFlag =
eatMoreFlagOnce || frame.offset == initialOffset && lastErrorPos > frame.offset;
final LighterASTNode latestDoneMarker =
(pinned || result) && (state.altMode || lastErrorPos > initialOffset) && eatMoreFlagOnce
? builder_.getLatestDoneMarker()
: null;
PsiBuilder.Marker extensionMarker = null;
IElementType extensionTokenType = null;
if (latestDoneMarker instanceof PsiBuilder.Marker) {
extensionMarker = ((PsiBuilder.Marker) latestDoneMarker).precede();
extensionTokenType = latestDoneMarker.getTokenType();
((PsiBuilder.Marker) latestDoneMarker).drop();
}
// advance to the last error pos
// skip tokens until lastErrorPos. parseAsTree might look better here...
int parenCount = 0;
while (eatMoreFlag && builder_.getCurrentOffset() < lastErrorPos) {
if (state.braces != null) {
if (builder_.getTokenType() == state.braces[0].getLeftBraceType()) parenCount++;
else if (builder_.getTokenType() == state.braces[0].getRightBraceType()) parenCount--;
}
builder_.advanceLexer();
eatMoreFlag = parenCount != 0 || eatMore.parse(builder_, frame.level + 1);
}
boolean errorReported = frame.errorReportedAt == initialOffset;
if (errorReported) {
if (eatMoreFlag) {
builder_.advanceLexer();
parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore);
}
} else if (eatMoreFlag) {
String tokenText = builder_.getTokenText();
String expectedText = state.getExpectedText(builder_);
PsiBuilder.Marker mark = builder_.mark();
builder_.advanceLexer();
final String gotText =
!expectedText.isEmpty() ? "got '" + tokenText + "'" : "'" + tokenText + "' unexpected";
mark.error(expectedText + gotText);
parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore);
errorReported = true;
} else if (eatMoreFlagOnce || (!result && frame.offset != builder_.getCurrentOffset())) {
reportError(state, builder_, true);
errorReported = true;
}
if (extensionMarker != null) {
extensionMarker.done(extensionTokenType);
}
state.suppressErrors = false;
if (errorReported || result) {
state.clearVariants(true, 0);
state.clearVariants(false, 0);
state.lastExpectedVariantOffset = -1;
}
if (!result && eatMoreFlagOnce && frame.offset != builder_.getCurrentOffset()) result = true;
} else if (!result && pinned && frame.errorReportedAt < 0) {
// do not report if there're errors after current offset
if (getLastVariantOffset(state, initialOffset) == initialOffset) {
// do not force, inner recoverRoot might have skipped some tokens
if (reportError(state, builder_, false)) {
frame.errorReportedAt = initialOffset;
}
}
}
// propagate errorReportedAt up the stack to avoid duplicate reporting
Frame prevFrame = state.levelCheck.isEmpty() ? null : state.levelCheck.getLast();
if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt)
prevFrame.errorReportedAt = frame.errorReportedAt;
state.FRAMES.recycle(frame);
return result;
}
