Ejemplo n.º 1
0
 /**
  * <strong>[11.8.3] Numeric Literals</strong>
  *
  * <pre>
  * NumericLiteral ::
  *     DecimalLiteral
  *     BinaryIntegerLiteral
  *     OctalIntegerLiteral
  *     HexIntegerLiteral
  * </pre>
  */
 private Token readNumberLiteral(int c) {
   if (c == '0') {
     int d = input.get();
     if (d == 'x' || d == 'X') {
       number = readHexIntegerLiteral();
     } else if (d == 'b' || d == 'B') {
       number = readBinaryIntegerLiteral();
     } else if (d == 'o' || d == 'O') {
       number = readOctalIntegerLiteral();
     } else if (isDecimalDigit(d)
         && parser.isEnabled(CompatibilityOption.LegacyOctalIntegerLiteral)) {
       input.unget(d);
       number = readLegacyOctalIntegerLiteral();
     } else {
       input.unget(d);
       number = readDecimalLiteral(c);
     }
   } else {
     number = readDecimalLiteral(c);
   }
   return Token.NUMBER;
 }
Ejemplo n.º 2
0
  /**
   * <strong>[11.5] Token</strong>
   *
   * <pre>
   * Token ::
   *     IdentifierName
   *     Punctuator
   *     NumericLiteral
   *     StringLiteral
   *     Template
   * </pre>
   */
  private Token scanToken() {
    TokenStreamInput input = this.input;

    int c;
    for (; ; ) {
      c = input.get();
      if (c == TokenStreamInput.EOF) {
        return Token.EOF;
      } else if (c <= 0x20) {
        if (c == 0x09 || c == 0x0B || c == 0x0C || c == 0x20) {
          // skip over whitespace
          continue;
        }
        if (c == '\n') {
          incrementLineAndUpdate();
          continue;
        }
        if (c == '\r') {
          match('\n');
          incrementLineAndUpdate();
          continue;
        }
      } else if (c >= 0xA0) {
        if (isWhitespace(c)) {
          // skip over whitespace
          continue;
        }
        if (isLineTerminator(c)) {
          incrementLineAndUpdate();
          continue;
        }
      }
      break;
    }
    updateSourcePosition();

    if (DEBUG) System.out.printf("scanToken() -> %c\n", (char) c);

    switch (c) {
      case '\'':
      case '"':
        return readString(c);
      case '0':
      case '1':
      case '2':
      case '3':
      case '4':
      case '5':
      case '6':
      case '7':
      case '8':
      case '9':
        return readNumberLiteral(c);
      case 'A':
      case 'B':
      case 'C':
      case 'D':
      case 'E':
      case 'F':
      case 'G':
      case 'H':
      case 'I':
      case 'J':
      case 'K':
      case 'L':
      case 'M':
      case 'N':
      case 'O':
      case 'P':
      case 'Q':
      case 'R':
      case 'S':
      case 'T':
      case 'U':
      case 'V':
      case 'W':
      case 'X':
      case 'Y':
      case 'Z':
      case 'a':
      case 'b':
      case 'c':
      case 'd':
      case 'e':
      case 'f':
      case 'g':
      case 'h':
      case 'i':
      case 'j':
      case 'k':
      case 'l':
      case 'm':
      case 'n':
      case 'o':
      case 'p':
      case 'q':
      case 'r':
      case 's':
      case 't':
      case 'u':
      case 'v':
      case 'w':
      case 'x':
      case 'y':
      case 'z':
      case '$':
      case '_':
        return readIdentifier(c);
      case '{':
        return Token.LC;
      case '}':
        return Token.RC;
      case '(':
        return Token.LP;
      case ')':
        return Token.RP;
      case '[':
        return Token.LB;
      case ']':
        return Token.RB;
      case '.':
        switch (input.peek(0)) {
          case '0':
          case '1':
          case '2':
          case '3':
          case '4':
          case '5':
          case '6':
          case '7':
          case '8':
          case '9':
            return readNumberLiteral(c);
          case '.':
            if (input.peek(1) == '.') {
              mustMatch('.');
              mustMatch('.');
              return Token.TRIPLE_DOT;
            }
        }
        return Token.DOT;
      case ';':
        return Token.SEMI;
      case ',':
        return Token.COMMA;
      case '~':
        return Token.BITNOT;
      case '?':
        return Token.HOOK;
      case ':':
        return Token.COLON;
      case '<':
        if (match('<')) {
          if (match('=')) {
            return Token.ASSIGN_SHL;
          } else {
            return Token.SHL;
          }
        } else if (match('=')) {
          return Token.LE;
        } else if (input.peek(0) == '!'
            && input.peek(1) == '-'
            && input.peek(2) == '-'
            && parser.isEnabled(CompatibilityOption.HTMLComments)) {
          // html start-comment
          mustMatch('!');
          mustMatch('-');
          mustMatch('-');
          readSingleComment();
          return Token.COMMENT;
        } else {
          return Token.LT;
        }
      case '>':
        if (match('>')) {
          if (match('>')) {
            if (match('=')) {
              return Token.ASSIGN_USHR;
            } else {
              return Token.USHR;
            }
          } else if (match('=')) {
            return Token.ASSIGN_SHR;
          } else {
            return Token.SHR;
          }
        } else if (match('=')) {
          return Token.GE;
        } else {
          return Token.GT;
        }
      case '=':
        if (match('=')) {
          if (match('=')) {
            return Token.SHEQ;
          } else {
            return Token.EQ;
          }
        } else if (match('>')) {
          return Token.ARROW;
        } else {
          return Token.ASSIGN;
        }
      case '!':
        if (match('=')) {
          if (match('=')) {
            return Token.SHNE;
          } else {
            return Token.NE;
          }
        } else {
          return Token.NOT;
        }
      case '+':
        if (match('+')) {
          return Token.INC;
        } else if (match('=')) {
          return Token.ASSIGN_ADD;
        } else {
          return Token.ADD;
        }
      case '-':
        if (match('-')) {
          if (input.peek(0) == '>'
              && hasLineTerminator
              && parser.isEnabled(CompatibilityOption.HTMLComments)) {
            // html end-comment at line start
            mustMatch('>');
            readSingleComment();
            return Token.COMMENT;
          }
          return Token.DEC;
        } else if (match('=')) {
          return Token.ASSIGN_SUB;
        } else {
          return Token.SUB;
        }
      case '*':
        if (match('=')) {
          return Token.ASSIGN_MUL;
        } else {
          return Token.MUL;
        }
      case '%':
        if (match('=')) {
          return Token.ASSIGN_MOD;
        } else {
          return Token.MOD;
        }
      case '/':
        if (match('=')) {
          return Token.ASSIGN_DIV;
        } else if (match('/')) {
          readSingleComment();
          return Token.COMMENT;
        } else if (match('*')) {
          readMultiComment();
          return Token.COMMENT;
        } else {
          return Token.DIV;
        }
      case '&':
        if (match('&')) {
          return Token.AND;
        } else if (match('=')) {
          return Token.ASSIGN_BITAND;
        } else {
          return Token.BITAND;
        }
      case '|':
        if (match('|')) {
          return Token.OR;
        } else if (match('=')) {
          return Token.ASSIGN_BITOR;
        } else {
          return Token.BITOR;
        }
      case '^':
        if (match('=')) {
          return Token.ASSIGN_BITXOR;
        } else {
          return Token.BITXOR;
        }
      case '`':
        return Token.TEMPLATE;
    }

    if (c == '\\') {
      mustMatch('u');
      c = readUnicode();
    }
    if (isIdentifierStart(c)) {
      return readIdentifier(c);
    }

    return Token.ERROR;
  }
Ejemplo n.º 3
0
  /**
   * <strong>[11.8.4] String Literals</strong>
   *
   * <pre>
   * StringLiteral ::
   *     " DoubleStringCharacters<sub>opt</sub> "
   *     ' SingleStringCharacters<sub>opt</sub> '
   * DoubleStringCharacters ::
   *     DoubleStringCharacter DoubleStringCharacters<sub>opt</sub>
   * SingleStringCharacters ::
   *     SingleStringCharacter SingleStringCharacters<sub>opt</sub>
   * DoubleStringCharacter ::
   *     SourceCharacter but not one of " or \ or LineTerminator
   *     \ EscapeSequence
   *     LineContinuation
   * SingleStringCharacter ::
   *     SourceCharacter but not one of ' or \ or LineTerminator
   *     \ EscapeSequence
   *     LineContinuation
   * LineContinuation ::
   *     \ LineTerminatorSequence
   * EscapeSequence ::
   *     CharacterEscapeSequence
   *     0  [lookahead &#x2209; DecimalDigit]
   *     HexEscapeSequence
   *     UnicodeEscapeSequence
   * CharacterEscapeSequence ::
   *     SingleEscapeCharacter
   *     NonEscapeCharacter
   * SingleEscapeCharacter ::  one of
   *     ' "  \  b f n r t v
   * NonEscapeCharacter ::
   *     SourceCharacter but not one of EscapeCharacter or LineTerminator
   * EscapeCharacter ::
   *     SingleEscapeCharacter
   *     DecimalDigit
   *     x
   *     u
   * HexEscapeSequence ::
   *     x HexDigit HexDigit
   * UnicodeEscapeSequence ::
   *     u HexDigit HexDigit HexDigit HexDigit
   *     u{ HexDigits }
   * </pre>
   *
   * <strong>[B.1.2] String Literals</strong>
   *
   * <pre>
   * EscapeSequence ::
   *     CharacterEscapeSequence
   *     OctalEscapeSequence
   *     HexEscapeSequence
   *     UnicodeEscapeSequence
   * </pre>
   */
  private Token readString(int quoteChar) {
    assert quoteChar == '"' || quoteChar == '\'';

    final int EOF = TokenStreamInput.EOF;
    TokenStreamInput input = this.input;
    int start = input.position();
    StringBuffer buffer = this.buffer();
    hasEscape = false;
    for (; ; ) {
      int c = input.get();
      if (c == EOF) {
        throw eofError(Messages.Key.UnterminatedStringLiteral);
      }
      if (c == quoteChar) {
        buffer.add(input.range(start, input.position() - 1));
        break;
      }
      if (isLineTerminator(c)) {
        throw error(Messages.Key.UnterminatedStringLiteral);
      }
      if (c != '\\') {
        continue;
      }
      buffer.add(input.range(start, input.position() - 1));
      hasEscape = true;
      c = input.get();
      if (isLineTerminator(c)) {
        // line continuation
        if (c == '\r' && match('\n')) {
          // \r\n sequence
        }
        incrementLine();
        start = input.position();
        continue;
      }
      // escape sequences
      switch (c) {
        case 'b':
          c = '\b';
          break;
        case 'f':
          c = '\f';
          break;
        case 'n':
          c = '\n';
          break;
        case 'r':
          c = '\r';
          break;
        case 't':
          c = '\t';
          break;
        case 'v':
          c = '\u000B';
          break;
        case 'x':
          c = (hexDigit(input.get()) << 4) | hexDigit(input.get());
          if (c < 0) {
            throw error(Messages.Key.InvalidHexEscape);
          }
          break;
        case 'u':
          c = readUnicode();
          break;
        case '0':
          if (isDecimalDigit(input.peek(0))) {
            if (!parser.isEnabled(CompatibilityOption.OctalEscapeSequence)) {
              throw error(Messages.Key.InvalidNULLEscape);
            }
            c = readOctalEscape(c);
          } else {
            c = '\0';
          }
          break;
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
          if (!parser.isEnabled(CompatibilityOption.OctalEscapeSequence)) {
            throw error(Messages.Key.StrictModeOctalEscapeSequence);
          }
          c = readOctalEscape(c);
          break;
        case '8':
        case '9':
          // FIXME: spec bug - undefined behaviour for \8 and \9
          if (!parser.isEnabled(CompatibilityOption.OctalEscapeSequence)) {
            throw error(Messages.Key.StrictModeOctalEscapeSequence);
          }
          // fall-through
        case '"':
        case '\'':
        case '\\':
        default:
          // fall-through
      }
      buffer.addCodepoint(c);
      start = input.position();
    }

    return Token.STRING;
  }