Ejemplo n.º 1
0
  public byte markTokensImpl(byte token, Segment line, int lineIndex) {
    char[] array = line.array;
    int offset = line.offset;
    int lastOffset = offset;
    int length = line.count + offset;

    if (SyntaxUtilities.regionMatches(true, line, offset, "rem")) {
      addToken(line.count, Token.COMMENT1);
      return Token.NULL;
    }

    loop:
    for (int i = offset; i < length; i++) {
      int i1 = (i + 1);

      switch (token) {
        case Token.NULL:
          switch (array[i]) {
            case '%':
              addToken(i - lastOffset, token);
              lastOffset = i;
              if (length - i <= 3 || array[i + 2] == ' ') {
                addToken(2, Token.KEYWORD2);
                i += 2;
                lastOffset = i;
              } else token = Token.KEYWORD2;
              break;
            case '"':
              addToken(i - lastOffset, token);
              token = Token.LITERAL1;
              lastOffset = i;
              break;
            case ':':
              if (i == offset) {
                addToken(line.count, Token.LABEL);
                lastOffset = length;
                break loop;
              }
              break;
            case ' ':
              if (lastOffset == offset) {
                addToken(i - lastOffset, Token.KEYWORD1);
                lastOffset = i;
              }
              break;
          }
          break;
        case Token.KEYWORD2:
          if (array[i] == '%') {
            addToken(i1 - lastOffset, token);
            token = Token.NULL;
            lastOffset = i1;
          }
          break;
        case Token.LITERAL1:
          if (array[i] == '"') {
            addToken(i1 - lastOffset, token);
            token = Token.NULL;
            lastOffset = i1;
          }
          break;
        default:
          throw new InternalError("Invalid state: " + token);
      }
    }

    if (lastOffset != length) {
      if (token != Token.NULL) token = Token.INVALID;
      else if (lastOffset == offset) token = Token.KEYWORD1;
      addToken(length - lastOffset, token);
    }
    return Token.NULL;
  }
Ejemplo n.º 2
0
  public byte markTokensImpl(byte _token, Segment line, int lineIndex) {
    char[] array = line.array;
    int offset = line.offset;
    token = _token;
    lastOffset = offset;
    lastKeyword = offset;
    matchChar = '\0';
    matchCharBracket = false;
    matchSpacesAllowed = false;
    int length = line.count + offset;

    if (token == Token.LITERAL1 && lineIndex != 0 && lineInfo[lineIndex - 1].obj != null) {
      String str = (String) lineInfo[lineIndex - 1].obj;
      if (str != null
          && str.length() == line.count
          && SyntaxUtilities.regionMatches(false, line, offset, str)) {
        addToken(line.count, token);
        return Token.NULL;
      } else {
        addToken(line.count, token);
        lineInfo[lineIndex].obj = str;
        return token;
      }
    }

    boolean backslash = false;
    loop:
    for (int i = offset; i < length; i++) {
      int i1 = (i + 1);

      char c = array[i];
      if (c == '\\') {
        backslash = !backslash;
        continue;
      }

      switch (token) {
        case Token.NULL:
          switch (c) {
            case '#':
              if (doKeyword(line, i, c)) break;
              if (backslash) backslash = false;
              else {
                addToken(i - lastOffset, token);
                addToken(length - i, Token.COMMENT1);
                lastOffset = lastKeyword = length;
                break loop;
              }
              break;
            case '=':
              backslash = false;
              if (i == offset) {
                token = Token.COMMENT2;
                addToken(length - i, token);
                lastOffset = lastKeyword = length;
                break loop;
              } else doKeyword(line, i, c);
              break;
            case '$':
            case '&':
            case '%':
            case '@':
              backslash = false;
              if (doKeyword(line, i, c)) break;
              if (length - i > 1) {
                if (c == '&' && (array[i1] == '&' || Character.isWhitespace(array[i1]))) i++;
                else {
                  addToken(i - lastOffset, token);
                  lastOffset = lastKeyword = i;
                  token = Token.KEYWORD2;
                }
              }
              break;
            case '"':
              if (doKeyword(line, i, c)) break;
              if (backslash) backslash = false;
              else {
                addToken(i - lastOffset, token);
                token = Token.LITERAL1;
                lineInfo[lineIndex].obj = null;
                lastOffset = lastKeyword = i;
              }
              break;
            case '\'':
              if (backslash) backslash = false;
              else {
                int oldLastKeyword = lastKeyword;
                if (doKeyword(line, i, c)) break;
                if (i != oldLastKeyword) break;
                addToken(i - lastOffset, token);
                token = Token.LITERAL2;
                lastOffset = lastKeyword = i;
              }
              break;
            case '`':
              if (doKeyword(line, i, c)) break;
              if (backslash) backslash = false;
              else {
                addToken(i - lastOffset, token);
                token = Token.OPERATOR;
                lastOffset = lastKeyword = i;
              }
              break;
            case '<':
              if (doKeyword(line, i, c)) break;
              if (backslash) backslash = false;
              else {
                if (length - i > 2 && array[i1] == '<' && !Character.isWhitespace(array[i + 2])) {
                  addToken(i - lastOffset, token);
                  lastOffset = lastKeyword = i;
                  token = Token.LITERAL1;
                  int len = length - (i + 2);
                  if (array[length - 1] == ';') len--;
                  lineInfo[lineIndex].obj = createReadinString(array, i + 2, len);
                }
              }
              break;
            case ':':
              backslash = false;
              if (doKeyword(line, i, c)) break;
              // Doesn't pick up all labels,
              // but at least doesn't mess up
              if (lastKeyword != 0) break;
              addToken(i1 - lastOffset, Token.LABEL);
              lastOffset = lastKeyword = i1;
              break;
            case '-':
              backslash = false;
              if (doKeyword(line, i, c)) break;
              if (i != lastKeyword || length - i <= 1) break;
              switch (array[i1]) {
                case 'r':
                case 'w':
                case 'x':
                case 'o':
                case 'R':
                case 'W':
                case 'X':
                case 'O':
                case 'e':
                case 'z':
                case 's':
                case 'f':
                case 'd':
                case 'l':
                case 'p':
                case 'S':
                case 'b':
                case 'c':
                case 't':
                case 'u':
                case 'g':
                case 'k':
                case 'T':
                case 'B':
                case 'M':
                case 'A':
                case 'C':
                  addToken(i - lastOffset, token);
                  addToken(2, Token.KEYWORD3);
                  lastOffset = lastKeyword = i + 2;
                  i++;
              }
              break;
            case '/':
            case '?':
              if (doKeyword(line, i, c)) break;
              if (length - i > 1) {
                backslash = false;
                char ch = array[i1];
                if (Character.isWhitespace(ch)) break;
                matchChar = c;
                matchSpacesAllowed = false;
                addToken(i - lastOffset, token);
                token = S_ONE;
                lastOffset = lastKeyword = i;
              }
              break;
            default:
              backslash = false;
              if (!Character.isLetterOrDigit(c) && c != '_') doKeyword(line, i, c);
              break;
          }
          break;
        case Token.KEYWORD2:
          backslash = false;
          // This test checks for an end-of-variable
          // condition
          if (!Character.isLetterOrDigit(c)
              && c != '_'
              && c != '#'
              && c != '\''
              && c != ':'
              && c != '&') {
            // If this is the first character
            // of the variable name ($'aaa)
            // ignore it
            if (i != offset && array[i - 1] == '$') {
              addToken(i1 - lastOffset, token);
              lastOffset = lastKeyword = i1;
            }
            // Otherwise, end of variable...
            else {
              addToken(i - lastOffset, token);
              lastOffset = lastKeyword = i;
              // Wind back so that stuff
              // like $hello$fred is picked
              // up
              i--;
              token = Token.NULL;
            }
          }
          break;
        case S_ONE:
        case S_TWO:
          if (backslash) backslash = false;
          else {
            if (matchChar == '\0') {
              if (Character.isWhitespace(matchChar) && !matchSpacesAllowed) break;
              else matchChar = c;
            } else {
              switch (matchChar) {
                case '(':
                  matchChar = ')';
                  matchCharBracket = true;
                  break;
                case '[':
                  matchChar = ']';
                  matchCharBracket = true;
                  break;
                case '{':
                  matchChar = '}';
                  matchCharBracket = true;
                  break;
                case '<':
                  matchChar = '>';
                  matchCharBracket = true;
                  break;
                default:
                  matchCharBracket = false;
                  break;
              }
              if (c != matchChar) break;
              if (token == S_TWO) {
                token = S_ONE;
                if (matchCharBracket) matchChar = '\0';
              } else {
                token = S_END;
                addToken(i1 - lastOffset, Token.LITERAL2);
                lastOffset = lastKeyword = i1;
              }
            }
          }
          break;
        case S_END:
          backslash = false;
          if (!Character.isLetterOrDigit(c) && c != '_') doKeyword(line, i, c);
          break;
        case Token.COMMENT2:
          backslash = false;
          if (i == offset) {
            addToken(line.count, token);
            if (length - i > 3 && SyntaxUtilities.regionMatches(false, line, offset, "=cut"))
              token = Token.NULL;
            lastOffset = lastKeyword = length;
            break loop;
          }
          break;
        case Token.LITERAL1:
          if (backslash) backslash = false;
          /*
           * else if(c == '$') backslash = true;
           */
          else if (c == '"') {
            addToken(i1 - lastOffset, token);
            token = Token.NULL;
            lastOffset = lastKeyword = i1;
          }
          break;
        case Token.LITERAL2:
          if (backslash) backslash = false;
          /*
           * else if(c == '$') backslash = true;
           */
          else if (c == '\'') {
            addToken(i1 - lastOffset, Token.LITERAL1);
            token = Token.NULL;
            lastOffset = lastKeyword = i1;
          }
          break;
        case Token.OPERATOR:
          if (backslash) backslash = false;
          else if (c == '`') {
            addToken(i1 - lastOffset, token);
            token = Token.NULL;
            lastOffset = lastKeyword = i1;
          }
          break;
        default:
          throw new InternalError("Invalid state: " + token);
      }
    }

    if (token == Token.NULL) doKeyword(line, length, '\0');

    switch (token) {
      case Token.KEYWORD2:
        addToken(length - lastOffset, token);
        token = Token.NULL;
        break;
      case Token.LITERAL2:
        addToken(length - lastOffset, Token.LITERAL1);
        break;
      case S_END:
        addToken(length - lastOffset, Token.LITERAL2);
        token = Token.NULL;
        break;
      case S_ONE:
      case S_TWO:
        addToken(length - lastOffset, Token.INVALID);
        token = Token.NULL;
        break;
      default:
        addToken(length - lastOffset, token);
        break;
    }
    return token;
  }
Ejemplo n.º 3
0
  public byte markTokensImpl(byte token, Segment line, int lineIndex) {
    char[] array = line.array;
    byte cmdState = 0; // 0 = space before command, 1 = inside
    // command, 2 = after command
    int offset = line.offset;
    int lastOffset = offset;
    int length = line.count + offset;

    if (token == Token.LITERAL1 && lineIndex != 0 && lineInfo[lineIndex - 1].obj != null) {
      String str = (String) lineInfo[lineIndex - 1].obj;
      if (str != null
          && str.length() == line.count
          && SyntaxUtilities.regionMatches(false, line, offset, str)) {
        addToken(line.count, Token.LITERAL1);
        return Token.NULL;
      } else {
        addToken(line.count, Token.LITERAL1);
        lineInfo[lineIndex].obj = str;
        return Token.LITERAL1;
      }
    }

    boolean backslash = false;
    loop:
    for (int i = offset; i < length; i++) {
      int i1 = (i + 1);

      char c = array[i];

      if (c == '\\') {
        backslash = !backslash;
        continue;
      }

      switch (token) {
        case Token.NULL:
          switch (c) {
            case ' ':
            case '\t':
            case '(':
            case ')':
              backslash = false;
              if (cmdState == 1 /* insideCmd */) {
                addToken(i - lastOffset, Token.KEYWORD1);
                lastOffset = i;
                cmdState = 2; /* afterCmd */
              }
              break;
            case '=':
              backslash = false;
              if (cmdState == 1 /* insideCmd */) {
                addToken(i - lastOffset, token);
                lastOffset = i;
                cmdState = 2; /* afterCmd */
              }
              break;
            case '&':
            case '|':
            case ';':
              if (backslash) backslash = false;
              else cmdState = 0; /* beforeCmd */
              break;
            case '#':
              if (backslash) backslash = false;
              else {
                addToken(i - lastOffset, token);
                addToken(length - i, Token.COMMENT1);
                lastOffset = length;
                break loop;
              }
              break;
            case '$':
              if (backslash) backslash = false;
              else {
                addToken(i - lastOffset, token);
                cmdState = 2; /* afterCmd */
                lastOffset = i;
                if (length - i >= 2) {
                  switch (array[i1]) {
                    case '(':
                      continue;
                    case '{':
                      token = LVARIABLE;
                      break;
                    default:
                      token = Token.KEYWORD2;
                      break;
                  }
                } else token = Token.KEYWORD2;
              }
              break;
            case '"':
              if (backslash) backslash = false;
              else {
                addToken(i - lastOffset, token);
                token = Token.LITERAL1;
                lineInfo[lineIndex].obj = null;
                cmdState = 2; /* afterCmd */
                lastOffset = i;
              }
              break;
            case '\'':
              if (backslash) backslash = false;
              else {
                addToken(i - lastOffset, token);
                token = Token.LITERAL2;
                cmdState = 2; /* afterCmd */
                lastOffset = i;
              }
              break;
            case '<':
              if (backslash) backslash = false;
              else {
                if (length - i > 1 && array[i1] == '<') {
                  addToken(i - lastOffset, token);
                  token = Token.LITERAL1;
                  lastOffset = i;
                  lineInfo[lineIndex].obj = new String(array, i + 2, length - (i + 2));
                }
              }
              break;
            default:
              backslash = false;
              if (Character.isLetter(c)) {
                if (cmdState == 0 /* beforeCmd */) {
                  addToken(i - lastOffset, token);
                  lastOffset = i;
                  cmdState++; /* insideCmd */
                }
              }
              break;
          }
          break;
        case Token.KEYWORD2:
          backslash = false;
          if (!Character.isLetterOrDigit(c) && c != '_') {
            if (i != offset && array[i - 1] == '$') {
              addToken(i1 - lastOffset, token);
              lastOffset = i1;
              token = Token.NULL;
              continue;
            } else {
              addToken(i - lastOffset, token);
              lastOffset = i;
              token = Token.NULL;
            }
          }
          break;
        case Token.LITERAL1:
          if (backslash) backslash = false;
          else if (c == '"') {
            addToken(i1 - lastOffset, token);
            cmdState = 2; /* afterCmd */
            lastOffset = i1;
            token = Token.NULL;
          } else backslash = false;
          break;
        case Token.LITERAL2:
          if (backslash) backslash = false;
          else if (c == '\'') {
            addToken(i1 - lastOffset, Token.LITERAL1);
            cmdState = 2; /* afterCmd */
            lastOffset = i1;
            token = Token.NULL;
          } else backslash = false;
          break;
        case LVARIABLE:
          backslash = false;
          if (c == '}') {
            addToken(i1 - lastOffset, Token.KEYWORD2);
            lastOffset = i1;
            token = Token.NULL;
          }
          break;
        default:
          throw new InternalError("Invalid state: " + token);
      }
    }

    switch (token) {
      case Token.NULL:
        if (cmdState == 1) addToken(length - lastOffset, Token.KEYWORD1);
        else addToken(length - lastOffset, token);
        break;
      case Token.LITERAL2:
        addToken(length - lastOffset, Token.LITERAL1);
        break;
      case Token.KEYWORD2:
        addToken(length - lastOffset, token);
        token = Token.NULL;
        break;
      case LVARIABLE:
        addToken(length - lastOffset, Token.INVALID);
        token = Token.NULL;
        break;
      default:
        addToken(length - lastOffset, token);
        break;
    }
    return token;
  }