@Override
public FlyweightCapableTreeStructure<LighterASTNode> parseContents(
LighterLazyParseableNode chameleon) {
final PsiBuilder builder = createBuilder(chameleon.getText());
parse(builder);
return builder.getLightTree();
}
public static boolean empty_element_parsed_guard_(PsiBuilder builder, String funcName, int pos) {
if (pos == current_position_(builder)) {
builder.error(
"Empty element parsed in '" + funcName + "' at offset " + builder.getCurrentOffset());
return false;
}
return true;
}
private static void addVariant(PsiBuilder builder, ErrorState state, Object o) {
builder.eof(); // skip whitespaces
addVariantInner(state, builder.rawTokenIndex(), o);
CompletionState completionState = state.completionState;
if (completionState != null && state.predicateSign) {
addCompletionVariant(builder, completionState, o);
}
}
private static void close_frame_impl_(
ErrorState state,
Frame frame,
PsiBuilder builder,
PsiBuilder.Marker marker,
IElementType elementType,
boolean result,
boolean pinned) {
if (elementType != null && marker != null) {
if ((frame.modifiers & _COLLAPSE_) != 0) {
PsiBuilderImpl.ProductionMarker last =
result || pinned
? (PsiBuilderImpl.ProductionMarker) builder.getLatestDoneMarker()
: null;
if (last != null
&& last.getStartIndex() == frame.position
&& state.typeExtends(last.getTokenType(), elementType)) {
IElementType resultType = last.getTokenType();
((PsiBuilder.Marker) last).drop();
marker.done(resultType);
return;
}
}
if (result || pinned) {
if ((frame.modifiers & _UPPER_) != 0) {
marker.drop();
frame.parentFrame.elementType = elementType;
} else if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
marker.done(elementType);
frame.leftMarker.precede().done(((LighterASTNode) frame.leftMarker).getTokenType());
frame.leftMarker.drop();
} else if ((frame.modifiers & _LEFT_) != 0 && frame.leftMarker != null) {
marker.drop();
frame.leftMarker.precede().done(elementType);
} else {
if (frame.level == 0) builder.eof(); // skip whitespaces
marker.done(elementType);
}
} else {
close_marker_impl_(frame, marker, null, false);
}
} else if (result || pinned) {
if (marker != null) marker.drop();
if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
frame.leftMarker.precede().done(((LighterASTNode) frame.leftMarker).getTokenType());
frame.leftMarker.drop();
}
} else {
close_marker_impl_(frame, marker, null, false);
}
}
public void parse(PsiBuilder builder) {
final PsiBuilder.Marker root = builder.mark();
PsiBuilder.Marker error = null;
while (!builder.eof()) {
final String token = builder.getTokenText();
if ("?".equals(token)) error = builder.mark();
builder.advanceLexer();
if (error != null) {
error.error("test error 2");
error = null;
}
}
root.done(this);
}
private static void doTest(
@NonNls final String text, final Parser parser, @NonNls final String expected) {
final PsiBuilder builder = createBuilder(text);
final PsiBuilder.Marker rootMarker = builder.mark();
parser.parse(builder);
rootMarker.done(ROOT);
// check light tree composition
final FlyweightCapableTreeStructure<LighterASTNode> lightTree = builder.getLightTree();
assertEquals(expected, DebugUtil.lightTreeToString(lightTree, false));
// verify that light tree can be taken multiple times
final FlyweightCapableTreeStructure<LighterASTNode> lightTree2 = builder.getLightTree();
assertEquals(expected, DebugUtil.lightTreeToString(lightTree2, false));
// check heavy tree composition
final ASTNode root = builder.getTreeBuilt();
assertEquals(expected, DebugUtil.nodeTreeToString(root, false));
// check heavy vs. light tree merging
final PsiBuilder builder2 = createBuilder(text);
final PsiBuilder.Marker rootMarker2 = builder2.mark();
parser.parse(builder2);
rootMarker2.done(ROOT);
DiffTree.diff(
new ASTStructure(root),
builder2.getLightTree(),
new ShallowNodeComparator<ASTNode, LighterASTNode>() {
@Override
public ThreeState deepEqual(ASTNode oldNode, LighterASTNode newNode) {
return ThreeState.UNSURE;
}
@Override
public boolean typesEqual(ASTNode oldNode, LighterASTNode newNode) {
return true;
}
@Override
public boolean hashCodesEqual(ASTNode oldNode, LighterASTNode newNode) {
return true;
}
},
new DiffTreeChangeBuilder<ASTNode, LighterASTNode>() {
@Override
public void nodeReplaced(@NotNull ASTNode oldChild, @NotNull LighterASTNode newChild) {
fail("replaced(" + oldChild + "," + newChild.getTokenType() + ")");
}
@Override
public void nodeDeleted(@NotNull ASTNode oldParent, @NotNull ASTNode oldNode) {
fail("deleted(" + oldParent + "," + oldNode + ")");
}
@Override
public void nodeInserted(
@NotNull ASTNode oldParent, @NotNull LighterASTNode newNode, int pos) {
fail("inserted(" + oldParent + "," + newNode.getTokenType() + ")");
}
});
}
public Frame init(
PsiBuilder builder, ErrorState state, int level_, int modifiers_, String name_) {
offset = builder.getCurrentOffset();
position = builder.rawTokenIndex();
level = level_;
modifiers = modifiers_;
name = name_;
variantCount = state.variants.size();
errorReportedAt = -1;
Frame prev = state.frameStack.peekLast();
errorReportedAtPrev = prev == null ? -1 : prev.errorReportedAt;
leftMarker = null;
return this;
}
private static boolean reportError(ErrorState state, PsiBuilder builder_, boolean force) {
String expectedText = state.getExpectedText(builder_);
boolean notEmpty = StringUtil.isNotEmpty(expectedText);
if (force || notEmpty) {
final String gotText =
builder_.eof()
? "unexpected end of file"
: notEmpty
? "got '" + builder_.getTokenText() + "'"
: "'" + builder_.getTokenText() + "' unexpected";
builder_.error(expectedText + gotText);
return true;
}
return false;
}
@Override
public ASTNode parseContents(final ASTNode chameleon) {
if (JavaParserDefinition.USE_NEW_PARSER) {
final PsiBuilder builder = JavaParserUtil.createBuilder(chameleon);
doParse(builder);
return builder.getTreeBuilt().getFirstChildNode();
}
FileElement node = (FileElement) chameleon;
final CharSequence seq = node.getChars();
final PsiManager manager = node.getManager();
final JavaLexer lexer = new JavaLexer(PsiUtil.getLanguageLevel(node.getPsi()));
return FileTextParsing.parseFileText(manager, lexer, seq, 0, seq.length(), node.getCharTable());
}
public static boolean nextTokenIsFast(PsiBuilder builder, IElementType... tokens) {
IElementType tokenType = builder.getTokenType();
for (IElementType token : tokens) {
if (token == tokenType) return true;
}
return false;
}
public static boolean consumeTokenFast(PsiBuilder builder, IElementType token) {
if (nextTokenIsFast(builder, token)) {
builder.advanceLexer();
return true;
}
return false;
}
public static boolean consumeToken(PsiBuilder builder_, IElementType token) {
if (nextTokenIsInner(builder_, token, true)) {
builder_.advanceLexer();
return true;
}
return false;
}
private static void enter_section_impl_(
PsiBuilder builder, int level, int modifiers, @Nullable String frameName) {
ErrorState state = ErrorState.get(builder);
Frame frame = state.FRAMES.alloc().init(builder, state, level, modifiers, frameName);
Frame prevFrame = state.frameStack.peekLast();
if (prevFrame != null && prevFrame.errorReportedAt > frame.position) {
// report error for previous unsuccessful frame
reportError(builder, state, frame, null, true, false);
}
if (((frame.modifiers & _LEFT_) | (frame.modifiers & _LEFT_INNER_)) != 0) {
PsiBuilder.Marker left = (PsiBuilder.Marker) builder.getLatestDoneMarker();
if (invalid_left_marker_guard_(builder, left, frameName)) {
frame.leftMarker = left;
}
}
state.frameStack.add(frame);
if ((modifiers & _AND_) != 0) {
if (state.predicateCount == 0 && !state.predicateSign) {
throw new AssertionError("Incorrect false predicate sign");
}
state.predicateCount++;
} else if ((modifiers & _NOT_) != 0) {
if (state.predicateCount == 0) {
state.predicateSign = false;
} else {
state.predicateSign = !state.predicateSign;
}
state.predicateCount++;
}
}
public static boolean recursion_guard_(PsiBuilder builder_, int level_, String funcName_) {
if (level_ > 1000) {
builder_.error("Maximum recursion level (" + 1000 + ") reached in " + funcName_);
return false;
}
return true;
}
public static void report_error_(PsiBuilder builder_) {
ErrorState state = ErrorState.get(builder_);
Frame frame = state.levelCheck.isEmpty() ? null : state.levelCheck.getLast();
if (frame == null) {
LOG.error("Unbalanced error section: got null , expected " + frame);
return;
}
int offset = builder_.getCurrentOffset();
if (frame.errorReportedAt < offset
&& getLastVariantOffset(state, builder_.getCurrentOffset()) <= offset) {
if (reportError(state, builder_, true)) {
frame.errorReportedAt = offset;
}
}
}
public static boolean replaceVariants(PsiBuilder builder_, int variantCount, String frameName) {
ErrorState state = ErrorState.get(builder_);
if (!state.suppressErrors && state.predicateCount < 2 && state.predicateSign) {
state.clearVariants(true, state.variants.size() - variantCount);
addVariantInner(state, builder_.getCurrentOffset(), frameName);
}
return true;
}
public String getExpectedText(PsiBuilder builder_) {
int offset = builder_.getCurrentOffset();
StringBuilder sb = new StringBuilder();
if (addExpected(sb, offset, true)) {
sb.append(" expected, ");
} else if (addExpected(sb, offset, false)) sb.append(" unexpected, ");
return sb.toString();
}
public static boolean consumeTokenFast(PsiBuilder builder, String text) {
int count = nextTokenIsFast(builder, text, ErrorState.get(builder).caseSensitive);
if (count > 0) {
while (count-- > 0) builder.advanceLexer();
return true;
}
return false;
}
public static boolean recursion_guard_(PsiBuilder builder, int level, String funcName) {
if (level > MAX_RECURSION_LEVEL) {
builder.error(
"Maximum recursion level (" + MAX_RECURSION_LEVEL + ") reached in '" + funcName + "'");
return false;
}
return true;
}
private static boolean reportError(
PsiBuilder builder,
ErrorState state,
Frame frame,
IElementType elementType,
boolean force,
boolean advance) {
String expectedText = state.getExpectedText(builder);
boolean notEmpty = StringUtil.isNotEmpty(expectedText);
if (force || notEmpty || advance) {
String gotText =
builder.eof()
? "unexpected end of file"
: notEmpty
? "got '" + builder.getTokenText() + "'"
: "'" + builder.getTokenText() + "' unexpected";
String message = expectedText + gotText;
if (advance) {
PsiBuilder.Marker mark = builder.mark();
builder.advanceLexer();
mark.error(message);
} else if (!force) {
PsiBuilder.Marker extensionMarker = null;
IElementType extensionTokenType = null;
PsiBuilderImpl.ProductionMarker latestDoneMarker =
elementType == null
? null
: (PsiBuilderImpl.ProductionMarker) builder.getLatestDoneMarker();
if (latestDoneMarker != null
&& frame.position >= latestDoneMarker.getStartIndex()
&& frame.position <= latestDoneMarker.getEndIndex()) {
extensionMarker = ((PsiBuilder.Marker) latestDoneMarker).precede();
extensionTokenType = latestDoneMarker.getTokenType();
((PsiBuilder.Marker) latestDoneMarker).drop();
}
builder.error(message);
if (extensionMarker != null) extensionMarker.done(extensionTokenType);
} else {
builder.error(message);
}
builder.eof(); // skip whitespaces
frame.errorReportedAt = builder.rawTokenIndex();
return true;
}
return false;
}
public static boolean consumeToken(PsiBuilder builder, IElementType token) {
addVariantSmart(builder, token, true);
if (nextTokenIsFast(builder, token)) {
builder.advanceLexer();
return true;
}
return false;
}
public String getExpectedText(PsiBuilder builder) {
int position = builder.rawTokenIndex();
StringBuilder sb = new StringBuilder();
if (addExpected(sb, position, true)) {
sb.append(" expected, ");
}
return sb.toString();
}
public static boolean nextTokenIsInner(PsiBuilder builder_, IElementType token, boolean force) {
ErrorState state = ErrorState.get(builder_);
if (state.completionState != null && !force) return true;
IElementType tokenType = builder_.getTokenType();
if (!state.suppressErrors && state.predicateCount < 2) {
addVariant(state, builder_, token);
}
return token == tokenType;
}
private static void addVariant(ErrorState state, PsiBuilder builder_, Object o) {
int offset = builder_.getCurrentOffset();
addVariantInner(state, offset, o);
CompletionState completionState = state.completionState;
if (completionState != null && state.predicateSign) {
addCompletionVariant(state, completionState, builder_, o, offset);
}
}
public static boolean consumeToken(PsiBuilder builder, String text, boolean caseSensitive) {
addVariantSmart(builder, text, true);
int count = nextTokenIsFast(builder, text, caseSensitive);
if (count > 0) {
while (count-- > 0) builder.advanceLexer();
return true;
}
return false;
}
private static void addCompletionVariant(
ErrorState state,
CompletionState completionState,
PsiBuilder builder_,
Object o,
int offset) {
boolean add = false;
int diff = completionState.offset - offset;
String text = completionState.convertItem(o);
int length = text == null ? 0 : text.length();
if (length == 0) return;
if (diff == 0) {
add = true;
} else if (diff > 0 && diff <= length) {
CharSequence fragment =
builder_.getOriginalText().subSequence(offset, completionState.offset);
add = StringUtil.startsWithIgnoreCase(text, fragment.toString());
} else if (diff < 0) {
for (int i = -1; ; i--) {
IElementType type = builder_.rawLookup(i);
int tokenStart = builder_.rawTokenTypeStart(i);
if (state.whitespaceTokens.contains(type) || state.commentTokens.contains(type)) {
diff = completionState.offset - tokenStart;
} else if (type != null && tokenStart < completionState.offset) {
CharSequence fragment =
builder_.getOriginalText().subSequence(tokenStart, completionState.offset);
if (StringUtil.startsWithIgnoreCase(text, fragment.toString())) {
diff = completionState.offset - tokenStart;
}
break;
} else break;
}
add = diff >= 0 && diff < length;
}
add =
add
&& length > 1
&& !(text.charAt(0) == '<' && text.charAt(length - 1) == '>')
&& !(text.charAt(0) == '\'' && text.charAt(length - 1) == '\'' && length < 5);
if (add) {
completionState.items.add(text);
}
}
private static boolean addVariantSmart(PsiBuilder builder, Object token, boolean force) {
ErrorState state = ErrorState.get(builder);
// skip FIRST check in completion mode
if (state.completionState != null && !force) return false;
builder.eof();
if (!state.suppressErrors && state.predicateCount < 2) {
addVariant(builder, state, token);
}
return true;
}
public static int nextTokenIsFast(PsiBuilder builder, String tokenText, boolean caseSensitive) {
CharSequence sequence = builder.getOriginalText();
int offset = builder.getCurrentOffset();
int endOffset = offset + tokenText.length();
CharSequence subSequence = sequence.subSequence(offset, Math.min(endOffset, sequence.length()));
if (!Comparing.equal(subSequence, tokenText, caseSensitive)) return 0;
int count = 0;
while (true) {
int nextOffset = builder.rawTokenTypeStart(++count);
if (nextOffset > endOffset) {
return -count;
} else if (nextOffset == endOffset) {
break;
}
}
return count;
}
public static boolean nextTokenIs(PsiBuilder builder, String frameName, IElementType... tokens) {
ErrorState state = ErrorState.get(builder);
if (state.completionState != null) return true;
boolean track = !state.suppressErrors && state.predicateCount < 2 && state.predicateSign;
if (!track) return nextTokenIsFast(builder, tokens);
IElementType tokenType = builder.getTokenType();
if (StringUtil.isNotEmpty(frameName)) {
addVariantInner(state, builder.rawTokenIndex(), frameName);
} else {
for (IElementType token : tokens) {
addVariant(builder, state, token);
}
}
if (tokenType == null) return false;
for (IElementType token : tokens) {
if (tokenType == token) return true;
}
return false;
}
public static void initState(
ErrorState state, PsiBuilder builder, IElementType root, TokenSet[] extendsSets) {
state.extendsSets = extendsSets;
PsiFile file = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY);
state.completionState = file == null ? null : file.getUserData(COMPLETION_STATE_KEY);
Language language = file == null ? root.getLanguage() : file.getLanguage();
state.caseSensitive = language.isCaseSensitive();
PairedBraceMatcher matcher = LanguageBraceMatching.INSTANCE.forLanguage(language);
state.braces = matcher == null ? null : matcher.getPairs();
if (state.braces != null && state.braces.length == 0) state.braces = null;
}