private void doSetText(final CharSequence text) { final TokenProcessor processor = createTokenProcessor(0); myLexer.start(text, 0, text.length(), myInitialState); mySegments.removeAll(); int i = 0; while (true) { final IElementType tokenType = myLexer.getTokenType(); if (tokenType == null) break; int data = packData(tokenType, myLexer.getState()); processor.addToken(i, myLexer.getTokenStart(), myLexer.getTokenEnd(), data, tokenType); i++; myLexer.advance(); } processor.finish(); if (myEditor != null && !ApplicationManager.getApplication().isHeadlessEnvironment()) { UIUtil.invokeLaterIfNeeded( new DumbAwareRunnable() { @Override public void run() { myEditor.repaint(0, text.length()); } }); } }
public double isBashFile(File file, String data, Project project) { ParserDefinition definition = LanguageParserDefinitions.INSTANCE.forLanguage(BashFileType.BASH_LANGUAGE); Lexer lexer = definition.createLexer(project); lexer.start(data); int tokenCount = 0; Set<IElementType> tokenSet = Sets.newHashSet(); Set<Integer> modeSet = Sets.newHashSet(); while (lexer.getTokenType() != BashTokenTypes.BAD_CHARACTER && lexer.getTokenType() != null) { tokenSet.add(lexer.getTokenType()); modeSet.add(lexer.getState()); lexer.advance(); tokenCount++; } double score = 0; if (lexer.getTokenType() == BashTokenTypes.BAD_CHARACTER) { score -= badCharacterWeight; } if (tokenCount > 4) { score += tokenLimitWeight; } score += Math.min(0.45, (double) tokenSet.size() * tokenWeight); score += Math.min(0.45, (double) modeSet.size() * modeWeight); return score; }
protected void checkCorrectRestart(String text) { Lexer mainLexer = createLexer(); String allTokens = printTokens(text, 0, mainLexer); Lexer auxLexer = createLexer(); auxLexer.start(text); while (true) { IElementType type = auxLexer.getTokenType(); if (type == null) { break; } if (auxLexer.getState() == 0) { int tokenStart = auxLexer.getTokenStart(); String subTokens = printTokens(text, tokenStart, mainLexer); if (!allTokens.endsWith(subTokens)) { assertEquals( "Restarting impossible from offset " + tokenStart + "; lexer state should not return 0 at this point", allTokens, subTokens); } } auxLexer.advance(); } }
public int getState() { locateToken(); int state = 0; if (myLayeredTagStack.size() > 0) state = 239; if (myXmlState != 0) state = 239; int scalaState = myScalaPlainLexer.getState(); if (scalaState != 0) state = 239; return state; }
public LexerEditorHighlighter( @NotNull SyntaxHighlighter highlighter, @NotNull EditorColorsScheme scheme) { myScheme = scheme; myLexer = highlighter.getHighlightingLexer(); myLexer.start(ArrayUtil.EMPTY_CHAR_SEQUENCE); myInitialState = myLexer.getState(); myHighlighter = highlighter; mySegments = createSegments(); }
private void locateToken() { if (myTokenType == null) { IElementType type = myCurrentLexer.getTokenType(); int start = myCurrentLexer.getTokenStart(); String tokenText = myCurrentLexer .getBufferSequence() .subSequence(start, myCurrentLexer.getTokenEnd()) .toString(); if (myCurrentLexer == myXmlLexer && xmlSteps == 0) { myCurrentLexer = myScalaPlainLexer; myCurrentLexer.start(getBufferSequence(), start, myXmlLexer.getBufferEnd(), 0); } --xmlSteps; if (type == SCALA_XML_CONTENT_START) { final XmlTagValidator xmlTagValidator = new XmlTagValidator(myCurrentLexer); if (!xmlTagValidator.validate()) { xmlSteps = xmlTagValidator.step; } myCurrentLexer = myXmlLexer; myXmlState = 0; myCurrentLexer.start(getBufferSequence(), start, myBufferEnd, 0); myLayeredTagStack.push(new Stack<MyOpenXmlTag>()); myLayeredTagStack.peek().push(new MyOpenXmlTag()); myTokenType = myCurrentLexer.getTokenType(); locateTextRange(); } else if (( /*type == XML_ATTRIBUTE_VALUE_TOKEN || */ type == XML_DATA_CHARACTERS) && // todo: Dafuq??? tokenText.startsWith("{") && !tokenText.startsWith("{{") && !inCdata) { myXmlState = myCurrentLexer.getState(); (myCurrentLexer = myScalaPlainLexer).start(getBufferSequence(), start, myBufferEnd, 0); locateTextRange(); myBraceStack.push(1); myTokenType = SCALA_IN_XML_INJECTION_START; } else if (type == ScalaTokenTypes.tRBRACE && myBraceStack.size() > 0) { int currentLayer = myBraceStack.pop(); if (currentLayer == 1) { locateTextRange(); (myCurrentLexer = myXmlLexer) .start(getBufferSequence(), start + 1, myBufferEnd, myXmlState); myTokenType = SCALA_IN_XML_INJECTION_END; } else { myBraceStack.push(--currentLayer); } } else if (type == ScalaTokenTypes.tLBRACE && myBraceStack.size() > 0) { int currentLayer = myBraceStack.pop(); myBraceStack.push(++currentLayer); } else if ((XML_START_TAG_START == type || XML_COMMENT_START == type || XML_CDATA_START == type || XML_PI_START == type) && !myLayeredTagStack.isEmpty()) { if (type == XML_CDATA_START) { inCdata = true; } myLayeredTagStack.peek().push(new MyOpenXmlTag()); } else if (XML_EMPTY_ELEMENT_END == type && !myLayeredTagStack.isEmpty() && !myLayeredTagStack.peek().isEmpty() && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) { myLayeredTagStack.peek().pop(); if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) { myLayeredTagStack.pop(); locateTextRange(); startScalaPlainLexer(start + 2); myTokenType = XML_EMPTY_ELEMENT_END; return; } } else if (XML_TAG_END == type && !myLayeredTagStack.isEmpty() && !myLayeredTagStack.peek().isEmpty()) { MyOpenXmlTag tag = myLayeredTagStack.peek().peek(); if (tag.state == TAG_STATE.UNDEFINED) { tag.state = TAG_STATE.NONEMPTY; } else if (tag.state == TAG_STATE.NONEMPTY) { myLayeredTagStack.peek().pop(); } if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) { myLayeredTagStack.pop(); locateTextRange(); startScalaPlainLexer(start + 1); myTokenType = XML_TAG_END; return; } } else if (XML_PI_END == type && !myLayeredTagStack.isEmpty() && !myLayeredTagStack.peek().isEmpty() && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) { myLayeredTagStack.peek().pop(); if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) { myLayeredTagStack.pop(); locateTextRange(); startScalaPlainLexer(start + 2); myTokenType = XML_PI_END; return; } } else if (XML_COMMENT_END == type && !myLayeredTagStack.isEmpty() && !myLayeredTagStack.peek().isEmpty() && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) { myLayeredTagStack.peek().pop(); if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) { myLayeredTagStack.pop(); locateTextRange(); startScalaPlainLexer(start + 3); myTokenType = XML_COMMENT_END; return; } } else if (XML_CDATA_END == type && !myLayeredTagStack.isEmpty() && !myLayeredTagStack.peek().isEmpty() && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) { inCdata = false; myLayeredTagStack.peek().pop(); if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) { myLayeredTagStack.pop(); locateTextRange(); startScalaPlainLexer(start + 3); myTokenType = XML_CDATA_END; return; } } else if (type == XML_DATA_CHARACTERS && tokenText.indexOf('{') != -1 && !inCdata) { int scalaToken = tokenText.indexOf('{'); while (scalaToken != -1 && scalaToken + 1 < tokenText.length() && tokenText.charAt(scalaToken + 1) == '{') scalaToken = tokenText.indexOf('{', scalaToken + 2); if (scalaToken != -1) { myTokenType = XML_DATA_CHARACTERS; myTokenStart = myCurrentLexer.getTokenStart(); myTokenEnd = myTokenStart + scalaToken; myCurrentLexer.start( getBufferSequence(), myTokenEnd, myBufferEnd, myCurrentLexer.getState()); } } else if ((type == XML_REAL_WHITE_SPACE || type == XML_WHITE_SPACE || type == TAG_WHITE_SPACE) && tokenText.matches("\\s*\n(\n|\\s)*")) { type = ScalaTokenTypes.tWHITE_SPACE_IN_LINE; } else if (!(type instanceof IXmlLeafElementType)) { ++xmlSteps; } if (myTokenType == null) { myTokenType = type; if (myTokenType == null) return; locateTextRange(); } // we have to advance current lexer only if we didn't start scala plain lexer on this // iteration // because of wrong behaviour of the latter ScalaPlainLexer myCurrentLexer.advance(); } }
@Override public synchronized void documentChanged(DocumentEvent e) { final Document document = e.getDocument(); if (document instanceof DocumentEx && ((DocumentEx) document).isInBulkUpdate()) { mySegments.removeAll(); return; } if (mySegments.getSegmentCount() == 0) { setText(document.getCharsSequence()); return; } CharSequence text = document.getCharsSequence(); int oldStartOffset = e.getOffset(); final int segmentIndex; try { segmentIndex = mySegments.findSegmentIndex(oldStartOffset) - 2; } catch (IndexOutOfBoundsException ex) { throw new IndexOutOfBoundsException(ex.getMessage() + " Lexer: " + myLexer); } final int oldStartIndex = Math.max(0, segmentIndex); int startIndex = oldStartIndex; int data; do { data = mySegments.getSegmentData(startIndex); if (isInitialState(data) || startIndex == 0) break; startIndex--; } while (true); int startOffset = mySegments.getSegmentStart(startIndex); int newEndOffset = e.getOffset() + e.getNewLength(); myLexer.start(text, startOffset, text.length(), myInitialState); int lastTokenStart = -1; int lastLexerState = -1; while (myLexer.getTokenType() != null) { if (startIndex >= oldStartIndex) break; int tokenStart = myLexer.getTokenStart(); int lexerState = myLexer.getState(); if (tokenStart == lastTokenStart && lexerState == lastLexerState) { throw new IllegalStateException( "Error while updating lexer: " + e + " document text: " + document.getText()); } int tokenEnd = myLexer.getTokenEnd(); data = packData(myLexer.getTokenType(), lexerState); if (mySegments.getSegmentStart(startIndex) != tokenStart || mySegments.getSegmentEnd(startIndex) != tokenEnd || mySegments.getSegmentData(startIndex) != data) { break; } startIndex++; myLexer.advance(); lastTokenStart = tokenStart; lastLexerState = lexerState; } startOffset = mySegments.getSegmentStart(startIndex); int repaintEnd = -1; int insertSegmentCount = 0; int oldEndIndex = -1; SegmentArrayWithData insertSegments = new SegmentArrayWithData(); while (myLexer.getTokenType() != null) { int tokenStart = myLexer.getTokenStart(); int lexerState = myLexer.getState(); if (tokenStart == lastTokenStart && lexerState == lastLexerState) { throw new IllegalStateException( "Error while updating lexer: " + e + " document text: " + document.getText()); } lastTokenStart = tokenStart; lastLexerState = lexerState; int tokenEnd = myLexer.getTokenEnd(); data = packData(myLexer.getTokenType(), lexerState); if (tokenStart >= newEndOffset && lexerState == myInitialState) { int shiftedTokenStart = tokenStart - e.getNewLength() + e.getOldLength(); int index = mySegments.findSegmentIndex(shiftedTokenStart); if (mySegments.getSegmentStart(index) == shiftedTokenStart && mySegments.getSegmentData(index) == data) { repaintEnd = tokenStart; oldEndIndex = index; break; } } insertSegments.setElementAt(insertSegmentCount, tokenStart, tokenEnd, data); insertSegmentCount++; myLexer.advance(); } final int shift = e.getNewLength() - e.getOldLength(); if (repaintEnd > 0) { while (insertSegmentCount > 0 && oldEndIndex > startIndex) { if (!segmentsEqual( mySegments, oldEndIndex - 1, insertSegments, insertSegmentCount - 1, shift)) { break; } insertSegmentCount--; oldEndIndex--; repaintEnd = insertSegments.getSegmentStart(insertSegmentCount); insertSegments.remove(insertSegmentCount, insertSegmentCount + 1); } } if (repaintEnd == -1) { repaintEnd = text.length(); } if (oldEndIndex < 0) { oldEndIndex = mySegments.getSegmentCount(); } mySegments.shiftSegments(oldEndIndex, shift); mySegments.replace(startIndex, oldEndIndex, insertSegments); if (insertSegmentCount == 0 || oldEndIndex == startIndex + 1 && insertSegmentCount == 1 && data == mySegments.getSegmentData(startIndex)) { return; } myEditor.repaint(startOffset, repaintEnd); }