Beispiel #1
0
    private boolean validate() {
      valLexer.start(lexer.getBufferSequence(), lexer.getTokenStart(), lexer.getBufferEnd(), 0);
      step = 0;

      boolean isCdata = valLexer.getTokenType() == XML_CDATA_START;
      int state = valLexer.getState();

      advanceLexer();
      step = 1;

      if (!isCdata) {
        while (canProcess()) {
          if (valLexer.getTokenType() == XML_DATA_CHARACTERS) {
            return xmlTokens.peekLast() == XML_EQ
                && valLexer.getTokenText().startsWith("{")
                && !valLexer.getTokenText().startsWith("{{");
          } else {
            advanceLexer();
          }
          state = valLexer.getState();
        }

        return validStopTokens.contains(valLexer.getTokenType());
      }

      if (valLexer.getTokenType() == XML_CDATA_END) return true;
      advanceLexer();
      return valLexer.getTokenType() == XML_CDATA_END;
    }
  public String transform(String testName, String[] data) throws Exception {
    String fileText = data[0];

    Lexer lexer = new ScalaLexer();
    lexer.start(fileText);

    StringBuilder buffer = new StringBuilder();

    IElementType type;
    while ((type = lexer.getTokenType()) != null) {
      CharSequence s = lexer.getBufferSequence();
      s = s.subSequence(lexer.getTokenStart(), lexer.getTokenEnd());
      buffer.append(type.toString()).append(" {").append(s).append("}");
      lexer.advance();
      if (lexer.getTokenType() != null) {
        buffer.append("\n");
      }
    }

    Console.println("------------------------ " + testName + " ------------------------");
    Console.println(buffer.toString());
    Console.println("");

    return buffer.toString();
  }
Beispiel #3
0
 private boolean checkNotNextXmlBegin(Lexer lexer) {
   String text = lexer.getBufferSequence().toString();
   int beginIndex = lexer.getTokenEnd();
   if (beginIndex < text.length()) {
     text = text.substring(beginIndex).trim();
     if (text.length() > 2) {
       text = text.substring(0, 2);
     }
     return !text.matches(XML_BEGIN_PATTERN);
   }
   return true;
 }
Beispiel #4
0
 public void restore(LexerPosition position) {
   MyPosition pos = (MyPosition) position;
   myBraceStack = pos.state.braceStack;
   myCurrentLexer = pos.state.currentLexer;
   myTokenStart = pos.getOffset();
   myTokenEnd = pos.end;
   myLayeredTagStack = pos.state.tagStack;
   myCurrentLexer.start(
       myCurrentLexer.getBufferSequence(),
       myTokenStart,
       myBufferEnd,
       myCurrentLexer instanceof XmlLexer ? pos.state.xmlState : 0);
 }
  private static String getTokenText(Lexer lexer) {
    final IElementType tokenType = lexer.getTokenType();
    if (tokenType instanceof TokenWrapper) {
      return ((TokenWrapper) tokenType).getValue();
    }

    String text =
        lexer
            .getBufferSequence()
            .subSequence(lexer.getTokenStart(), lexer.getTokenEnd())
            .toString();
    text = StringUtil.replace(text, "\n", "\\n");
    return text;
  }
  protected OuterLanguageElementImpl createOuterLanguageElement(
      final Lexer lexer, final CharTable table, final IElementType outerElementType) {
    final CharSequence buffer = lexer.getBufferSequence();
    final int tokenStart = lexer.getTokenStart();
    if (tokenStart < 0 || tokenStart > buffer.length()) {
      LOG.error("Invalid start: " + tokenStart + "; " + lexer);
    }
    final int tokenEnd = lexer.getTokenEnd();
    if (tokenEnd < 0 || tokenEnd > buffer.length()) {
      LOG.error("Invalid end: " + tokenEnd + "; " + lexer);
    }

    return new OuterLanguageElementImpl(
        outerElementType, table.intern(buffer, tokenStart, tokenEnd));
  }
Beispiel #7
0
  private void locateToken() {
    if (myTokenType == null) {
      IElementType type = myCurrentLexer.getTokenType();
      int start = myCurrentLexer.getTokenStart();
      String tokenText =
          myCurrentLexer
              .getBufferSequence()
              .subSequence(start, myCurrentLexer.getTokenEnd())
              .toString();
      if (myCurrentLexer == myXmlLexer && xmlSteps == 0) {
        myCurrentLexer = myScalaPlainLexer;
        myCurrentLexer.start(getBufferSequence(), start, myXmlLexer.getBufferEnd(), 0);
      }

      --xmlSteps;
      if (type == SCALA_XML_CONTENT_START) {
        final XmlTagValidator xmlTagValidator = new XmlTagValidator(myCurrentLexer);
        if (!xmlTagValidator.validate()) {
          xmlSteps = xmlTagValidator.step;
        }

        myCurrentLexer = myXmlLexer;
        myXmlState = 0;
        myCurrentLexer.start(getBufferSequence(), start, myBufferEnd, 0);
        myLayeredTagStack.push(new Stack<MyOpenXmlTag>());
        myLayeredTagStack.peek().push(new MyOpenXmlTag());
        myTokenType = myCurrentLexer.getTokenType();
        locateTextRange();
      } else if ((
          /*type == XML_ATTRIBUTE_VALUE_TOKEN || */ type == XML_DATA_CHARACTERS)
          && // todo: Dafuq???
          tokenText.startsWith("{")
          && !tokenText.startsWith("{{")
          && !inCdata) {
        myXmlState = myCurrentLexer.getState();
        (myCurrentLexer = myScalaPlainLexer).start(getBufferSequence(), start, myBufferEnd, 0);
        locateTextRange();
        myBraceStack.push(1);
        myTokenType = SCALA_IN_XML_INJECTION_START;
      } else if (type == ScalaTokenTypes.tRBRACE && myBraceStack.size() > 0) {
        int currentLayer = myBraceStack.pop();
        if (currentLayer == 1) {
          locateTextRange();
          (myCurrentLexer = myXmlLexer)
              .start(getBufferSequence(), start + 1, myBufferEnd, myXmlState);
          myTokenType = SCALA_IN_XML_INJECTION_END;
        } else {
          myBraceStack.push(--currentLayer);
        }
      } else if (type == ScalaTokenTypes.tLBRACE && myBraceStack.size() > 0) {
        int currentLayer = myBraceStack.pop();
        myBraceStack.push(++currentLayer);
      } else if ((XML_START_TAG_START == type
              || XML_COMMENT_START == type
              || XML_CDATA_START == type
              || XML_PI_START == type)
          && !myLayeredTagStack.isEmpty()) {
        if (type == XML_CDATA_START) {
          inCdata = true;
        }
        myLayeredTagStack.peek().push(new MyOpenXmlTag());
      } else if (XML_EMPTY_ELEMENT_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()
          && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {

        myLayeredTagStack.peek().pop();
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 2);
          myTokenType = XML_EMPTY_ELEMENT_END;
          return;
        }
      } else if (XML_TAG_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()) {
        MyOpenXmlTag tag = myLayeredTagStack.peek().peek();
        if (tag.state == TAG_STATE.UNDEFINED) {
          tag.state = TAG_STATE.NONEMPTY;
        } else if (tag.state == TAG_STATE.NONEMPTY) {
          myLayeredTagStack.peek().pop();
        }
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 1);
          myTokenType = XML_TAG_END;
          return;
        }
      } else if (XML_PI_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()
          && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {

        myLayeredTagStack.peek().pop();
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 2);
          myTokenType = XML_PI_END;
          return;
        }
      } else if (XML_COMMENT_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()
          && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {

        myLayeredTagStack.peek().pop();
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 3);
          myTokenType = XML_COMMENT_END;
          return;
        }
      } else if (XML_CDATA_END == type
          && !myLayeredTagStack.isEmpty()
          && !myLayeredTagStack.peek().isEmpty()
          && myLayeredTagStack.peek().peek().state == TAG_STATE.UNDEFINED) {
        inCdata = false;
        myLayeredTagStack.peek().pop();
        if (myLayeredTagStack.peek().isEmpty() && checkNotNextXmlBegin(myCurrentLexer)) {
          myLayeredTagStack.pop();
          locateTextRange();
          startScalaPlainLexer(start + 3);
          myTokenType = XML_CDATA_END;
          return;
        }
      } else if (type == XML_DATA_CHARACTERS && tokenText.indexOf('{') != -1 && !inCdata) {
        int scalaToken = tokenText.indexOf('{');
        while (scalaToken != -1
            && scalaToken + 1 < tokenText.length()
            && tokenText.charAt(scalaToken + 1) == '{')
          scalaToken = tokenText.indexOf('{', scalaToken + 2);
        if (scalaToken != -1) {
          myTokenType = XML_DATA_CHARACTERS;
          myTokenStart = myCurrentLexer.getTokenStart();
          myTokenEnd = myTokenStart + scalaToken;
          myCurrentLexer.start(
              getBufferSequence(), myTokenEnd, myBufferEnd, myCurrentLexer.getState());
        }
      } else if ((type == XML_REAL_WHITE_SPACE
              || type == XML_WHITE_SPACE
              || type == TAG_WHITE_SPACE)
          && tokenText.matches("\\s*\n(\n|\\s)*")) {
        type = ScalaTokenTypes.tWHITE_SPACE_IN_LINE;
      } else if (!(type instanceof IXmlLeafElementType)) {
        ++xmlSteps;
      }
      if (myTokenType == null) {
        myTokenType = type;
        if (myTokenType == null) return;
        locateTextRange();
      }
      // we have to advance current lexer only if we didn't start scala plain lexer on this
      // iteration
      // because of wrong behaviour of the latter ScalaPlainLexer
      myCurrentLexer.advance();
    }
  }