private void expect(String string, Token... tokens) {
    System.out.println("Tokenizing [" + string + "]");
    ExprLexer tokenizer = new ExprLexer(string);
    int expectedIndex = 0;
    while (!tokenizer.isEndOfInput()) {
      String expectedString;
      Token expected;
      if (expectedIndex < tokens.length) {
        expected = tokens[expectedIndex];
        expectedString = expected.toString();
      } else {
        expected = null;
        expectedString = "<NOTHING>";
      }
      Token actual = tokenizer.next();
      System.out.println(
          String.format("Expected: %15s, got %s", expectedString, actual.toString()));
      if (expected != null) {
        assertEquals("tokenStart", expected.getTokenStart(), actual.getTokenStart());
        assertEquals("text", expected.getString(), actual.getString());
        assertEquals("type", expected.getType(), actual.getType());

        if (!expected.equals(actual)) {
          System.err.println("Unexpected result!");
          throw new AssertionError();
        }
      }
      expectedIndex++;
    }
  }
 private boolean isSingeLineComment() {
   if (m_CurrChar.getString().equals("/") && m_NextChar.getString().equals("/")) {
     m_InComment = true;
     return true;
   }
   return false;
 }
 public Token next() {
   try {
     if (m_NextNext.isEmpty() == false) {
       Token next = m_NextNext.get(0);
       m_NextNext.remove(0);
       m_Ret = new StringBuilder();
       return next;
     }
     if (m_CurrChar.isImmutable()) {
       Token ret1 = m_CurrChar;
       read();
       m_Ret = new StringBuilder();
       return ret1;
     }
     while (shouldTrim(m_CurrChar.getString())) read();
     if (m_CurrChar.isImmutable()) {
       Token ret1 = m_CurrChar;
       read();
       m_Ret = new StringBuilder();
       return ret1;
     }
     Token ret = doNext();
     while (shouldTrim(m_CurrChar.getString())) read();
     m_Ret = new StringBuilder();
     return ret;
   } catch (Exception ex) {
     m_EndOfStream = true;
     if (m_Ret == null) m_Ret = new StringBuilder();
     String ret_string = m_Ret.toString();
     Token ret = new Token(ret_string.trim());
     return ret;
   }
 }
 private boolean isSpecialEndToken() {
   String curr;
   if (m_SpecialEndToken.length() == 2) {
     curr = m_CurrChar.getString() + m_NextChar.getString();
   } else {
     curr = m_CurrChar.getString();
   }
   if (curr.equals(m_SpecialEndToken)) {
     if (m_SpecialEndToken.equals("*/")) {
       m_InComment = false;
     }
     return true;
   }
   return false;
 }
 private boolean isSpecialStartToken() {
   String two = m_CurrChar.getString() + m_NextChar.getString();
   if (two.equals("/*")) {
     m_InComment = true;
     m_SpecialEndToken = "*/";
     return true;
   } else if (m_CurrChar.getString().equals("\"") && m_InComment == false) {
     m_SpecialEndToken = "\"";
     return true;
   } else if (m_CurrChar.getString().equals("'") && m_InComment == false) {
     m_SpecialEndToken = "\'";
     return true;
   }
   return false;
 }
Example #6
0
 @Override
 public String toString() {
   StringBuilder sb = new StringBuilder();
   for (Token t : tokens)
     sb.append(t.toString()).append(" String:").append(t.getString()).append("\n");
   return sb.toString();
 }
Example #7
0
 public void lex() {
   try {
     tok = split.getToken();
     text = tok.getString();
     val = tok.getType();
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
 public void process(JCas jCas, ResultSpecification rs) throws AnnotatorProcessException {
   JFSIndexRepository indexRep = jCas.getJFSIndexRepository();
   FSIndex tokenIndex = indexRep.getAnnotationIndex(Token.type);
   Iterator tokens = tokenIndex.iterator();
   while (tokens.hasNext()) {
     Token tok = (Token) tokens.next();
     System.out.print("Token: String=\"" + tok.getString() + "\", ");
     System.out.print("Kind=\"" + tok.getKind() + "\", ");
     System.out.println("Orth=\"" + tok.getOrth() + "\"");
   }
 }
 private Token readSpecial() throws Exception {
   String ret = null;
   try {
     ret = m_CurrChar.getString();
     read();
     while (isSpecialEndToken() == false) {
       ret += m_CurrChar.getString();
       read();
     }
     read();
     if (m_SpecialEndToken.length() == 2) read();
     ret += m_SpecialEndToken;
     Token token = new Token(ret);
     token.setImmutable();
     return token;
   } catch (Exception ex) {
     Token token = new Token(ret);
     token.setImmutable();
     return token;
   }
 }
 private Token eatLine() throws Exception {
   m_Ret = new StringBuilder();
   m_Ret.append(m_CurrChar.getString());
   List<Token> all_ret = new ArrayList<Token>();
   boolean is_immutable = false;
   if (isSingeLineComment()) is_immutable = true;
   while (true) {
     read();
     if (isSpecialStartToken()) {
       Token curr = new Token(m_Ret.toString());
       if (is_immutable) curr.setImmutable();
       all_ret.add(curr);
       all_ret.add(readSpecial());
       is_immutable = false;
       m_Ret = new StringBuilder();
     } else if (m_InComment == false && isSingeLineComment()) {
       Token curr = new Token(m_Ret.toString());
       is_immutable = true;
       all_ret.add(curr);
       m_Ret = new StringBuilder();
     } else if (isNewLine(m_CurrChar.getString())) {
       Token curr = new Token(m_Ret.toString());
       if (is_immutable) curr.setImmutable();
       all_ret.add(curr);
       Token newline = new Token("\n");
       newline.setNewLine();
       all_ret.add(newline);
       Token ret = all_ret.get(0);
       all_ret.remove(0);
       m_NextNext.addAll(all_ret);
       is_immutable = false;
       // ignore # lines like they are comments
       // ret.setComment();
       return ret;
     }
     m_Ret.append(m_CurrChar.getString());
   }
 }
Example #11
0
  @Override
  public Expr parse(MagpieParser parser, Token token) {
    switch (token.getType()) {
      case BOOL:
        return Expr.bool(token.getPosition(), token.getBool());
      case INT:
        return Expr.int_(token.getPosition(), token.getInt());
      case NOTHING:
        return Expr.nothing(token.getPosition());
      case STRING:
        return Expr.string(token.getPosition(), token.getString());
    }

    throw new ParseException("Unexpected token type for literal.");
  }
 private Token doNext() throws Exception {
   if (isSingleLineToken()) {
     Token token = eatLine();
     m_InComment = false;
     token.setImmutable();
     return token;
   }
   if (isSpecialStartToken()) {
     return readSpecial();
   } else {
     m_Ret = new StringBuilder();
     while (isSpecialStartToken() == false && isSingleLineToken() == false) {
       m_Ret.append(m_CurrChar.getString());
       read();
     }
     Token ret = new Token(m_Ret.toString());
     return ret;
   }
 }
Example #13
0
  public int getPrecedence(Token token) {
    int precedence = 0;

    // A reserved word can't start an infix expression. Prevents us from
    // parsing a keyword as an identifier.
    if (isReserved(token)) return 0;

    // If we have a prefix parser for this token's name, then that takes
    // precedence. Prevents us from parsing a keyword as an identifier.
    if ((token.getValue() instanceof String) && mPrefixParsers.isReserved(token.getString()))
      return 0;

    InfixParser parser = mInfixParsers.get(token);
    if (parser != null) {
      precedence = parser.getPrecedence();
    }

    return precedence;
  }
 private boolean isSingleLineToken() {
   if (m_CurrChar.getString().equals("#")) return true;
   if (isSingeLineComment()) return true;
   return false;
 }
Example #15
0
 /**
  * Gets whether or not this token is a reserved word. Reserved words like "else" and "then" are
  * claimed for special use by mixfix parsers, so can't be parsed on their own.
  *
  * @param token The token to test
  * @return True if the token is a reserved name token.
  */
 public boolean isReserved(Token token) {
   return (token.getType() == TokenType.NAME) && isReserved(token.getString());
 }