private void doSetText(final CharSequence text) {
    final TokenProcessor processor = createTokenProcessor(0);
    myLexer.start(text, 0, text.length(), myInitialState);
    mySegments.removeAll();
    int i = 0;
    while (true) {
      final IElementType tokenType = myLexer.getTokenType();
      if (tokenType == null) break;

      int data = packData(tokenType, myLexer.getState());
      processor.addToken(i, myLexer.getTokenStart(), myLexer.getTokenEnd(), data, tokenType);
      i++;
      myLexer.advance();
    }
    processor.finish();

    if (myEditor != null && !ApplicationManager.getApplication().isHeadlessEnvironment()) {
      UIUtil.invokeLaterIfNeeded(
          new DumbAwareRunnable() {
            @Override
            public void run() {
              myEditor.repaint(0, text.length());
            }
          });
    }
  }
  public double isBashFile(File file, String data, Project project) {
    ParserDefinition definition =
        LanguageParserDefinitions.INSTANCE.forLanguage(BashFileType.BASH_LANGUAGE);

    Lexer lexer = definition.createLexer(project);
    lexer.start(data);

    int tokenCount = 0;
    Set<IElementType> tokenSet = Sets.newHashSet();
    Set<Integer> modeSet = Sets.newHashSet();
    while (lexer.getTokenType() != BashTokenTypes.BAD_CHARACTER && lexer.getTokenType() != null) {
      tokenSet.add(lexer.getTokenType());
      modeSet.add(lexer.getState());

      lexer.advance();
      tokenCount++;
    }

    double score = 0;
    if (lexer.getTokenType() == BashTokenTypes.BAD_CHARACTER) {
      score -= badCharacterWeight;
    }

    if (tokenCount > 4) {
      score += tokenLimitWeight;
    }

    score += Math.min(0.45, (double) tokenSet.size() * tokenWeight);
    score += Math.min(0.45, (double) modeSet.size() * modeWeight);

    return score;
  }
  protected void checkCorrectRestart(String text) {
    Lexer mainLexer = createLexer();
    String allTokens = printTokens(text, 0, mainLexer);

    Lexer auxLexer = createLexer();
    auxLexer.start(text);
    while (true) {
      IElementType type = auxLexer.getTokenType();
      if (type == null) {
        break;
      }
      if (auxLexer.getState() == 0) {
        int tokenStart = auxLexer.getTokenStart();
        String subTokens = printTokens(text, tokenStart, mainLexer);
        if (!allTokens.endsWith(subTokens)) {
          assertEquals(
              "Restarting impossible from offset "
                  + tokenStart
                  + "; lexer state should not return 0 at this point",
              allTokens,
              subTokens);
        }
      }
      auxLexer.advance();
    }
  }
Example #4
0
 public int getState() {
   locateToken();
   int state = 0;
   if (myLayeredTagStack.size() > 0) state = 239;
   if (myXmlState != 0) state = 239;
   int scalaState = myScalaPlainLexer.getState();
   if (scalaState != 0) state = 239;
   return state;
 }
 public LexerEditorHighlighter(
     @NotNull SyntaxHighlighter highlighter, @NotNull EditorColorsScheme scheme) {
   myScheme = scheme;
   myLexer = highlighter.getHighlightingLexer();
   myLexer.start(ArrayUtil.EMPTY_CHAR_SEQUENCE);
   myInitialState = myLexer.getState();
   myHighlighter = highlighter;
   mySegments = createSegments();
 }
Example #6
0
  private void locateToken() {
    if (myTokenType == null) {
      IElementType type = myCurrentLexer.getTokenType();
      int start = myCurrentLexer.getTokenStart();
      String tokenText =
          myCurrentLexer
              .getBufferSequence()
              .subSequence(start, myCurrentLexer.getTokenEnd())
              .toString();
      if (myCurrentLexer == myXmlLexer && xmlSteps == 0) {
        myCurrentLexer = myScalaPlainLexer;
        myCurrentLexer.start(getBufferSequence(), start, myXmlLexer.getBufferEnd(), 0);
      }

      --xmlSteps;
      if (type == SCALA_XML_CONTENT_START) {
        final XmlTagValidator xmlTagValidator = new XmlTagValidator(myCurrentLexer);
        if (!xmlTagValidator.validate()) {
          xmlSteps = xmlTagValidator.step;
        }

        myCurrentLexer = myXmlLexer;
        myXmlState = 0;
        myCurrentLexer.start(getBufferSequence(), start, myBufferEnd, 0);
        myLayeredTagStack.push(new Stack<MyOpenXmlTag>());
        myLayeredTagStack.peek().push(new MyOpenXmlTag());
        myTokenType = myCurrentLexer.getTokenType();
        locateTextRange();
      } else if ((
          /*type == XML_ATTRIBUTE_VALUE_TOKEN || */ type == XML_DATA_CHARACTERS)
          && // todo: Dafuq???
          tokenText.startsWith("{")
          && !tokenText.startsWith("{{")
          && !inCdata) {
        myXmlState = myCurrentLexer.getState();
        (myCurrentLexer = myScalaPlainLexer).start(getBufferSequence(), start, myBufferEnd, 0);
        locateTextRange();
        myBraceStack.push(1);
        myTokenType = SCALA_IN_XML_INJECTION_START;
      } else if (type == ScalaTokenTypes.tRBRACE && myBraceStack.size() > 0) {
        int currentLayer = myBraceStack.pop();
        if (currentLayer == 1) {
          locateTextRange();
          (myCurrentLexer = myXmlLexer)
              .start(getBufferSequence(), start + 1, myBufferEnd, myXmlState);
          myTokenType = SCALA_IN_XML_INJECTION_END;
        } else {
          myBraceStack.push(--currentLayer);
        }
      } else if (type == ScalaTokenTypes.tLBRACE && myBraceStack.size() > 0) {
        int currentLayer = myBraceStack.pop();
        myBraceStack.push(++currentLayer);
      } else if ((XML_START_TAG_START == type
              || XML_COMMENT_START == type
              || XML_CDATA_START == type
              || XML_PI_START == type)
          && !myLayeredTagStack.isEmpty()) {
        if (type == XML_CDATA_START) {
          inCdata = true;
        }
        myLayeredTagStack.peek().push(new MyOpenXmlTag());
      } else if (XML_EMPTY_ELEMENT_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()
          && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {

        myLayeredTagStack.peek().pop();
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 2);
          myTokenType = XML_EMPTY_ELEMENT_END;
          return;
        }
      } else if (XML_TAG_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()) {
        MyOpenXmlTag tag = myLayeredTagStack.peek().peek();
        if (tag.state == TAG_STATE.UNDEFINED) {
          tag.state = TAG_STATE.NONEMPTY;
        } else if (tag.state == TAG_STATE.NONEMPTY) {
          myLayeredTagStack.peek().pop();
        }
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 1);
          myTokenType = XML_TAG_END;
          return;
        }
      } else if (XML_PI_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()
          && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {

        myLayeredTagStack.peek().pop();
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 2);
          myTokenType = XML_PI_END;
          return;
        }
      } else if (XML_COMMENT_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()
          && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {

        myLayeredTagStack.peek().pop();
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 3);
          myTokenType = XML_COMMENT_END;
          return;
        }
      } else if (XML_CDATA_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()
          && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {
        inCdata = false;
        myLayeredTagStack.peek().pop();
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 3);
          myTokenType = XML_CDATA_END;
          return;
        }
      } else if (type == XML_DATA_CHARACTERS && tokenText.indexOf('{') != -1 && !inCdata) {
        int scalaToken = tokenText.indexOf('{');
        while (scalaToken != -1
            && scalaToken + 1 < tokenText.length()
            && tokenText.charAt(scalaToken + 1) == '{')
          scalaToken = tokenText.indexOf('{', scalaToken + 2);
        if (scalaToken != -1) {
          myTokenType = XML_DATA_CHARACTERS;
          myTokenStart = myCurrentLexer.getTokenStart();
          myTokenEnd = myTokenStart + scalaToken;
          myCurrentLexer.start(
              getBufferSequence(), myTokenEnd, myBufferEnd, myCurrentLexer.getState());
        }
      } else if ((type == XML_REAL_WHITE_SPACE
              || type == XML_WHITE_SPACE
              || type == TAG_WHITE_SPACE)
          && tokenText.matches("\\s*\n(\n|\\s)*")) {
        type = ScalaTokenTypes.tWHITE_SPACE_IN_LINE;
      } else if (!(type instanceof IXmlLeafElementType)) {
        ++xmlSteps;
      }
      if (myTokenType == null) {
        myTokenType = type;
        if (myTokenType == null) return;
        locateTextRange();
      }
      // we have to advance current lexer only if we didn't start scala plain lexer on this
      // iteration
      // because of wrong behaviour of the latter ScalaPlainLexer
      myCurrentLexer.advance();
    }
  }
  @Override
  public synchronized void documentChanged(DocumentEvent e) {
    final Document document = e.getDocument();

    if (document instanceof DocumentEx && ((DocumentEx) document).isInBulkUpdate()) {
      mySegments.removeAll();
      return;
    }

    if (mySegments.getSegmentCount() == 0) {
      setText(document.getCharsSequence());
      return;
    }

    CharSequence text = document.getCharsSequence();
    int oldStartOffset = e.getOffset();

    final int segmentIndex;
    try {
      segmentIndex = mySegments.findSegmentIndex(oldStartOffset) - 2;
    } catch (IndexOutOfBoundsException ex) {
      throw new IndexOutOfBoundsException(ex.getMessage() + " Lexer: " + myLexer);
    }
    final int oldStartIndex = Math.max(0, segmentIndex);
    int startIndex = oldStartIndex;

    int data;
    do {
      data = mySegments.getSegmentData(startIndex);
      if (isInitialState(data) || startIndex == 0) break;
      startIndex--;
    } while (true);

    int startOffset = mySegments.getSegmentStart(startIndex);
    int newEndOffset = e.getOffset() + e.getNewLength();

    myLexer.start(text, startOffset, text.length(), myInitialState);

    int lastTokenStart = -1;
    int lastLexerState = -1;

    while (myLexer.getTokenType() != null) {
      if (startIndex >= oldStartIndex) break;

      int tokenStart = myLexer.getTokenStart();
      int lexerState = myLexer.getState();

      if (tokenStart == lastTokenStart && lexerState == lastLexerState) {
        throw new IllegalStateException(
            "Error while updating lexer: " + e + " document text: " + document.getText());
      }

      int tokenEnd = myLexer.getTokenEnd();
      data = packData(myLexer.getTokenType(), lexerState);
      if (mySegments.getSegmentStart(startIndex) != tokenStart
          || mySegments.getSegmentEnd(startIndex) != tokenEnd
          || mySegments.getSegmentData(startIndex) != data) {
        break;
      }
      startIndex++;
      myLexer.advance();
      lastTokenStart = tokenStart;
      lastLexerState = lexerState;
    }

    startOffset = mySegments.getSegmentStart(startIndex);
    int repaintEnd = -1;
    int insertSegmentCount = 0;
    int oldEndIndex = -1;
    SegmentArrayWithData insertSegments = new SegmentArrayWithData();

    while (myLexer.getTokenType() != null) {
      int tokenStart = myLexer.getTokenStart();
      int lexerState = myLexer.getState();

      if (tokenStart == lastTokenStart && lexerState == lastLexerState) {
        throw new IllegalStateException(
            "Error while updating lexer: " + e + " document text: " + document.getText());
      }

      lastTokenStart = tokenStart;
      lastLexerState = lexerState;

      int tokenEnd = myLexer.getTokenEnd();
      data = packData(myLexer.getTokenType(), lexerState);
      if (tokenStart >= newEndOffset && lexerState == myInitialState) {
        int shiftedTokenStart = tokenStart - e.getNewLength() + e.getOldLength();
        int index = mySegments.findSegmentIndex(shiftedTokenStart);
        if (mySegments.getSegmentStart(index) == shiftedTokenStart
            && mySegments.getSegmentData(index) == data) {
          repaintEnd = tokenStart;
          oldEndIndex = index;
          break;
        }
      }
      insertSegments.setElementAt(insertSegmentCount, tokenStart, tokenEnd, data);
      insertSegmentCount++;
      myLexer.advance();
    }

    final int shift = e.getNewLength() - e.getOldLength();
    if (repaintEnd > 0) {
      while (insertSegmentCount > 0 && oldEndIndex > startIndex) {
        if (!segmentsEqual(
            mySegments, oldEndIndex - 1, insertSegments, insertSegmentCount - 1, shift)) {
          break;
        }
        insertSegmentCount--;
        oldEndIndex--;
        repaintEnd = insertSegments.getSegmentStart(insertSegmentCount);
        insertSegments.remove(insertSegmentCount, insertSegmentCount + 1);
      }
    }

    if (repaintEnd == -1) {
      repaintEnd = text.length();
    }

    if (oldEndIndex < 0) {
      oldEndIndex = mySegments.getSegmentCount();
    }
    mySegments.shiftSegments(oldEndIndex, shift);
    mySegments.replace(startIndex, oldEndIndex, insertSegments);

    if (insertSegmentCount == 0
        || oldEndIndex == startIndex + 1
            && insertSegmentCount == 1
            && data == mySegments.getSegmentData(startIndex)) {
      return;
    }

    myEditor.repaint(startOffset, repaintEnd);
  }