private void doSetText(final CharSequence text) {
final TokenProcessor processor = createTokenProcessor(0);
myLexer.start(text, 0, text.length(), myInitialState);
mySegments.removeAll();
int i = 0;
while (true) {
final IElementType tokenType = myLexer.getTokenType();
if (tokenType == null) break;
int data = packData(tokenType, myLexer.getState());
processor.addToken(i, myLexer.getTokenStart(), myLexer.getTokenEnd(), data, tokenType);
i++;
myLexer.advance();
}
processor.finish();
if (myEditor != null && !ApplicationManager.getApplication().isHeadlessEnvironment()) {
UIUtil.invokeLaterIfNeeded(
new DumbAwareRunnable() {
@Override
public void run() {
myEditor.repaint(0, text.length());
}
});
}
}
public double isBashFile(File file, String data, Project project) {
ParserDefinition definition =
LanguageParserDefinitions.INSTANCE.forLanguage(BashFileType.BASH_LANGUAGE);
Lexer lexer = definition.createLexer(project);
lexer.start(data);
int tokenCount = 0;
Set<IElementType> tokenSet = Sets.newHashSet();
Set<Integer> modeSet = Sets.newHashSet();
while (lexer.getTokenType() != BashTokenTypes.BAD_CHARACTER && lexer.getTokenType() != null) {
tokenSet.add(lexer.getTokenType());
modeSet.add(lexer.getState());
lexer.advance();
tokenCount++;
}
double score = 0;
if (lexer.getTokenType() == BashTokenTypes.BAD_CHARACTER) {
score -= badCharacterWeight;
}
if (tokenCount > 4) {
score += tokenLimitWeight;
}
score += Math.min(0.45, (double) tokenSet.size() * tokenWeight);
score += Math.min(0.45, (double) modeSet.size() * modeWeight);
return score;
}
protected void checkCorrectRestart(String text) {
Lexer mainLexer = createLexer();
String allTokens = printTokens(text, 0, mainLexer);
Lexer auxLexer = createLexer();
auxLexer.start(text);
while (true) {
IElementType type = auxLexer.getTokenType();
if (type == null) {
break;
}
if (auxLexer.getState() == 0) {
int tokenStart = auxLexer.getTokenStart();
String subTokens = printTokens(text, tokenStart, mainLexer);
if (!allTokens.endsWith(subTokens)) {
assertEquals(
"Restarting impossible from offset "
+ tokenStart
+ "; lexer state should not return 0 at this point",
allTokens,
subTokens);
}
}
auxLexer.advance();
}
}
public int getState() {
locateToken();
int state = 0;
if (myLayeredTagStack.size() > 0) state = 239;
if (myXmlState != 0) state = 239;
int scalaState = myScalaPlainLexer.getState();
if (scalaState != 0) state = 239;
return state;
}
public LexerEditorHighlighter(
@NotNull SyntaxHighlighter highlighter, @NotNull EditorColorsScheme scheme) {
myScheme = scheme;
myLexer = highlighter.getHighlightingLexer();
myLexer.start(ArrayUtil.EMPTY_CHAR_SEQUENCE);
myInitialState = myLexer.getState();
myHighlighter = highlighter;
mySegments = createSegments();
}
private void locateToken() {
if (myTokenType == null) {
IElementType type = myCurrentLexer.getTokenType();
int start = myCurrentLexer.getTokenStart();
String tokenText =
myCurrentLexer
.getBufferSequence()
.subSequence(start, myCurrentLexer.getTokenEnd())
.toString();
if (myCurrentLexer == myXmlLexer && xmlSteps == 0) {
myCurrentLexer = myScalaPlainLexer;
myCurrentLexer.start(getBufferSequence(), start, myXmlLexer.getBufferEnd(), 0);
}
--xmlSteps;
if (type == SCALA_XML_CONTENT_START) {
final XmlTagValidator xmlTagValidator = new XmlTagValidator(myCurrentLexer);
if (!xmlTagValidator.validate()) {
xmlSteps = xmlTagValidator.step;
}
myCurrentLexer = myXmlLexer;
myXmlState = 0;
myCurrentLexer.start(getBufferSequence(), start, myBufferEnd, 0);
myLayeredTagStack.push(new Stack<MyOpenXmlTag>());
myLayeredTagStack.peek().push(new MyOpenXmlTag());
myTokenType = myCurrentLexer.getTokenType();
locateTextRange();
} else if ((
/*type == XML_ATTRIBUTE_VALUE_TOKEN || */ type == XML_DATA_CHARACTERS)
&& // todo: Dafuq???
tokenText.startsWith("{")
&& !tokenText.startsWith("{{")
&& !inCdata) {
myXmlState = myCurrentLexer.getState();
(myCurrentLexer = myScalaPlainLexer).start(getBufferSequence(), start, myBufferEnd, 0);
locateTextRange();
myBraceStack.push(1);
myTokenType = SCALA_IN_XML_INJECTION_START;
} else if (type == ScalaTokenTypes.tRBRACE && myBraceStack.size() > 0) {
int currentLayer = myBraceStack.pop();
if (currentLayer == 1) {
locateTextRange();
(myCurrentLexer = myXmlLexer)
.start(getBufferSequence(), start + 1, myBufferEnd, myXmlState);
myTokenType = SCALA_IN_XML_INJECTION_END;
} else {
myBraceStack.push(--currentLayer);
}
} else if (type == ScalaTokenTypes.tLBRACE && myBraceStack.size() > 0) {
int currentLayer = myBraceStack.pop();
myBraceStack.push(++currentLayer);
} else if ((XML_START_TAG_START == type
|| XML_COMMENT_START == type
|| XML_CDATA_START == type
|| XML_PI_START == type)
&& !myLayeredTagStack.isEmpty()) {
if (type == XML_CDATA_START) {
inCdata = true;
}
myLayeredTagStack.peek().push(new MyOpenXmlTag());
} else if (XML_EMPTY_ELEMENT_END == type
&& !myLayeredTagStack.isEmpty()
&& !myLayeredTagStack.peek().isEmpty()
&& myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {
myLayeredTagStack.peek().pop();
if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
myLayeredTagStack.pop();
locateTextRange();
startScalaPlainLexer(start + 2);
myTokenType = XML_EMPTY_ELEMENT_END;
return;
}
} else if (XML_TAG_END == type
&& !myLayeredTagStack.isEmpty()
&& !myLayeredTagStack.peek().isEmpty()) {
MyOpenXmlTag tag = myLayeredTagStack.peek().peek();
if (tag.state == TAG_STATE.UNDEFINED) {
tag.state = TAG_STATE.NONEMPTY;
} else if (tag.state == TAG_STATE.NONEMPTY) {
myLayeredTagStack.peek().pop();
}
if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
myLayeredTagStack.pop();
locateTextRange();
startScalaPlainLexer(start + 1);
myTokenType = XML_TAG_END;
return;
}
} else if (XML_PI_END == type
&& !myLayeredTagStack.isEmpty()
&& !myLayeredTagStack.peek().isEmpty()
&& myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {
myLayeredTagStack.peek().pop();
if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
myLayeredTagStack.pop();
locateTextRange();
startScalaPlainLexer(start + 2);
myTokenType = XML_PI_END;
return;
}
} else if (XML_COMMENT_END == type
&& !myLayeredTagStack.isEmpty()
&& !myLayeredTagStack.peek().isEmpty()
&& myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {
myLayeredTagStack.peek().pop();
if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
myLayeredTagStack.pop();
locateTextRange();
startScalaPlainLexer(start + 3);
myTokenType = XML_COMMENT_END;
return;
}
} else if (XML_CDATA_END == type
&& !myLayeredTagStack.isEmpty()
&& !myLayeredTagStack.peek().isEmpty()
&& myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {
inCdata = false;
myLayeredTagStack.peek().pop();
if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
myLayeredTagStack.pop();
locateTextRange();
startScalaPlainLexer(start + 3);
myTokenType = XML_CDATA_END;
return;
}
} else if (type == XML_DATA_CHARACTERS && tokenText.indexOf('{') != -1 && !inCdata) {
int scalaToken = tokenText.indexOf('{');
while (scalaToken != -1
&& scalaToken + 1 < tokenText.length()
&& tokenText.charAt(scalaToken + 1) == '{')
scalaToken = tokenText.indexOf('{', scalaToken + 2);
if (scalaToken != -1) {
myTokenType = XML_DATA_CHARACTERS;
myTokenStart = myCurrentLexer.getTokenStart();
myTokenEnd = myTokenStart + scalaToken;
myCurrentLexer.start(
getBufferSequence(), myTokenEnd, myBufferEnd, myCurrentLexer.getState());
}
} else if ((type == XML_REAL_WHITE_SPACE
|| type == XML_WHITE_SPACE
|| type == TAG_WHITE_SPACE)
&& tokenText.matches("\\s*\n(\n|\\s)*")) {
type = ScalaTokenTypes.tWHITE_SPACE_IN_LINE;
} else if (!(type instanceof IXmlLeafElementType)) {
++xmlSteps;
}
if (myTokenType == null) {
myTokenType = type;
if (myTokenType == null) return;
locateTextRange();
}
// we have to advance current lexer only if we didn't start scala plain lexer on this
// iteration
// because of wrong behaviour of the latter ScalaPlainLexer
myCurrentLexer.advance();
}
}
@Override
public synchronized void documentChanged(DocumentEvent e) {
final Document document = e.getDocument();
if (document instanceof DocumentEx && ((DocumentEx) document).isInBulkUpdate()) {
mySegments.removeAll();
return;
}
if (mySegments.getSegmentCount() == 0) {
setText(document.getCharsSequence());
return;
}
CharSequence text = document.getCharsSequence();
int oldStartOffset = e.getOffset();
final int segmentIndex;
try {
segmentIndex = mySegments.findSegmentIndex(oldStartOffset) - 2;
} catch (IndexOutOfBoundsException ex) {
throw new IndexOutOfBoundsException(ex.getMessage() + " Lexer: " + myLexer);
}
final int oldStartIndex = Math.max(0, segmentIndex);
int startIndex = oldStartIndex;
int data;
do {
data = mySegments.getSegmentData(startIndex);
if (isInitialState(data) || startIndex == 0) break;
startIndex--;
} while (true);
int startOffset = mySegments.getSegmentStart(startIndex);
int newEndOffset = e.getOffset() + e.getNewLength();
myLexer.start(text, startOffset, text.length(), myInitialState);
int lastTokenStart = -1;
int lastLexerState = -1;
while (myLexer.getTokenType() != null) {
if (startIndex >= oldStartIndex) break;
int tokenStart = myLexer.getTokenStart();
int lexerState = myLexer.getState();
if (tokenStart == lastTokenStart && lexerState == lastLexerState) {
throw new IllegalStateException(
"Error while updating lexer: " + e + " document text: " + document.getText());
}
int tokenEnd = myLexer.getTokenEnd();
data = packData(myLexer.getTokenType(), lexerState);
if (mySegments.getSegmentStart(startIndex) != tokenStart
|| mySegments.getSegmentEnd(startIndex) != tokenEnd
|| mySegments.getSegmentData(startIndex) != data) {
break;
}
startIndex++;
myLexer.advance();
lastTokenStart = tokenStart;
lastLexerState = lexerState;
}
startOffset = mySegments.getSegmentStart(startIndex);
int repaintEnd = -1;
int insertSegmentCount = 0;
int oldEndIndex = -1;
SegmentArrayWithData insertSegments = new SegmentArrayWithData();
while (myLexer.getTokenType() != null) {
int tokenStart = myLexer.getTokenStart();
int lexerState = myLexer.getState();
if (tokenStart == lastTokenStart && lexerState == lastLexerState) {
throw new IllegalStateException(
"Error while updating lexer: " + e + " document text: " + document.getText());
}
lastTokenStart = tokenStart;
lastLexerState = lexerState;
int tokenEnd = myLexer.getTokenEnd();
data = packData(myLexer.getTokenType(), lexerState);
if (tokenStart >= newEndOffset && lexerState == myInitialState) {
int shiftedTokenStart = tokenStart - e.getNewLength() + e.getOldLength();
int index = mySegments.findSegmentIndex(shiftedTokenStart);
if (mySegments.getSegmentStart(index) == shiftedTokenStart
&& mySegments.getSegmentData(index) == data) {
repaintEnd = tokenStart;
oldEndIndex = index;
break;
}
}
insertSegments.setElementAt(insertSegmentCount, tokenStart, tokenEnd, data);
insertSegmentCount++;
myLexer.advance();
}
final int shift = e.getNewLength() - e.getOldLength();
if (repaintEnd > 0) {
while (insertSegmentCount > 0 && oldEndIndex > startIndex) {
if (!segmentsEqual(
mySegments, oldEndIndex - 1, insertSegments, insertSegmentCount - 1, shift)) {
break;
}
insertSegmentCount--;
oldEndIndex--;
repaintEnd = insertSegments.getSegmentStart(insertSegmentCount);
insertSegments.remove(insertSegmentCount, insertSegmentCount + 1);
}
}
if (repaintEnd == -1) {
repaintEnd = text.length();
}
if (oldEndIndex < 0) {
oldEndIndex = mySegments.getSegmentCount();
}
mySegments.shiftSegments(oldEndIndex, shift);
mySegments.replace(startIndex, oldEndIndex, insertSegments);
if (insertSegmentCount == 0
|| oldEndIndex == startIndex + 1
&& insertSegmentCount == 1
&& data == mySegments.getSegmentData(startIndex)) {
return;
}
myEditor.repaint(startOffset, repaintEnd);
}