private void expect(String string, Token... tokens) { System.out.println("Tokenizing [" + string + "]"); ExprLexer tokenizer = new ExprLexer(string); int expectedIndex = 0; while (!tokenizer.isEndOfInput()) { String expectedString; Token expected; if (expectedIndex < tokens.length) { expected = tokens[expectedIndex]; expectedString = expected.toString(); } else { expected = null; expectedString = "<NOTHING>"; } Token actual = tokenizer.next(); System.out.println( String.format("Expected: %15s, got %s", expectedString, actual.toString())); if (expected != null) { assertEquals("tokenStart", expected.getTokenStart(), actual.getTokenStart()); assertEquals("text", expected.getString(), actual.getString()); assertEquals("type", expected.getType(), actual.getType()); if (!expected.equals(actual)) { System.err.println("Unexpected result!"); throw new AssertionError(); } } expectedIndex++; } }
private boolean isSingeLineComment() { if (m_CurrChar.getString().equals("/") && m_NextChar.getString().equals("/")) { m_InComment = true; return true; } return false; }
public Token next() { try { if (m_NextNext.isEmpty() == false) { Token next = m_NextNext.get(0); m_NextNext.remove(0); m_Ret = new StringBuilder(); return next; } if (m_CurrChar.isImmutable()) { Token ret1 = m_CurrChar; read(); m_Ret = new StringBuilder(); return ret1; } while (shouldTrim(m_CurrChar.getString())) read(); if (m_CurrChar.isImmutable()) { Token ret1 = m_CurrChar; read(); m_Ret = new StringBuilder(); return ret1; } Token ret = doNext(); while (shouldTrim(m_CurrChar.getString())) read(); m_Ret = new StringBuilder(); return ret; } catch (Exception ex) { m_EndOfStream = true; if (m_Ret == null) m_Ret = new StringBuilder(); String ret_string = m_Ret.toString(); Token ret = new Token(ret_string.trim()); return ret; } }
private boolean isSpecialEndToken() { String curr; if (m_SpecialEndToken.length() == 2) { curr = m_CurrChar.getString() + m_NextChar.getString(); } else { curr = m_CurrChar.getString(); } if (curr.equals(m_SpecialEndToken)) { if (m_SpecialEndToken.equals("*/")) { m_InComment = false; } return true; } return false; }
private boolean isSpecialStartToken() { String two = m_CurrChar.getString() + m_NextChar.getString(); if (two.equals("/*")) { m_InComment = true; m_SpecialEndToken = "*/"; return true; } else if (m_CurrChar.getString().equals("\"") && m_InComment == false) { m_SpecialEndToken = "\""; return true; } else if (m_CurrChar.getString().equals("'") && m_InComment == false) { m_SpecialEndToken = "\'"; return true; } return false; }
@Override public String toString() { StringBuilder sb = new StringBuilder(); for (Token t : tokens) sb.append(t.toString()).append(" String:").append(t.getString()).append("\n"); return sb.toString(); }
public void lex() { try { tok = split.getToken(); text = tok.getString(); val = tok.getType(); } catch (Exception e) { e.printStackTrace(); } }
public void process(JCas jCas, ResultSpecification rs) throws AnnotatorProcessException { JFSIndexRepository indexRep = jCas.getJFSIndexRepository(); FSIndex tokenIndex = indexRep.getAnnotationIndex(Token.type); Iterator tokens = tokenIndex.iterator(); while (tokens.hasNext()) { Token tok = (Token) tokens.next(); System.out.print("Token: String=\"" + tok.getString() + "\", "); System.out.print("Kind=\"" + tok.getKind() + "\", "); System.out.println("Orth=\"" + tok.getOrth() + "\""); } }
private Token readSpecial() throws Exception { String ret = null; try { ret = m_CurrChar.getString(); read(); while (isSpecialEndToken() == false) { ret += m_CurrChar.getString(); read(); } read(); if (m_SpecialEndToken.length() == 2) read(); ret += m_SpecialEndToken; Token token = new Token(ret); token.setImmutable(); return token; } catch (Exception ex) { Token token = new Token(ret); token.setImmutable(); return token; } }
private Token eatLine() throws Exception { m_Ret = new StringBuilder(); m_Ret.append(m_CurrChar.getString()); List<Token> all_ret = new ArrayList<Token>(); boolean is_immutable = false; if (isSingeLineComment()) is_immutable = true; while (true) { read(); if (isSpecialStartToken()) { Token curr = new Token(m_Ret.toString()); if (is_immutable) curr.setImmutable(); all_ret.add(curr); all_ret.add(readSpecial()); is_immutable = false; m_Ret = new StringBuilder(); } else if (m_InComment == false && isSingeLineComment()) { Token curr = new Token(m_Ret.toString()); is_immutable = true; all_ret.add(curr); m_Ret = new StringBuilder(); } else if (isNewLine(m_CurrChar.getString())) { Token curr = new Token(m_Ret.toString()); if (is_immutable) curr.setImmutable(); all_ret.add(curr); Token newline = new Token("\n"); newline.setNewLine(); all_ret.add(newline); Token ret = all_ret.get(0); all_ret.remove(0); m_NextNext.addAll(all_ret); is_immutable = false; // ignore # lines like they are comments // ret.setComment(); return ret; } m_Ret.append(m_CurrChar.getString()); } }
@Override public Expr parse(MagpieParser parser, Token token) { switch (token.getType()) { case BOOL: return Expr.bool(token.getPosition(), token.getBool()); case INT: return Expr.int_(token.getPosition(), token.getInt()); case NOTHING: return Expr.nothing(token.getPosition()); case STRING: return Expr.string(token.getPosition(), token.getString()); } throw new ParseException("Unexpected token type for literal."); }
private Token doNext() throws Exception { if (isSingleLineToken()) { Token token = eatLine(); m_InComment = false; token.setImmutable(); return token; } if (isSpecialStartToken()) { return readSpecial(); } else { m_Ret = new StringBuilder(); while (isSpecialStartToken() == false && isSingleLineToken() == false) { m_Ret.append(m_CurrChar.getString()); read(); } Token ret = new Token(m_Ret.toString()); return ret; } }
public int getPrecedence(Token token) { int precedence = 0; // A reserved word can't start an infix expression. Prevents us from // parsing a keyword as an identifier. if (isReserved(token)) return 0; // If we have a prefix parser for this token's name, then that takes // precedence. Prevents us from parsing a keyword as an identifier. if ((token.getValue() instanceof String) && mPrefixParsers.isReserved(token.getString())) return 0; InfixParser parser = mInfixParsers.get(token); if (parser != null) { precedence = parser.getPrecedence(); } return precedence; }
private boolean isSingleLineToken() { if (m_CurrChar.getString().equals("#")) return true; if (isSingeLineComment()) return true; return false; }
/** * Gets whether or not this token is a reserved word. Reserved words like "else" and "then" are * claimed for special use by mixfix parsers, so can't be parsed on their own. * * @param token The token to test * @return True if the token is a reserved name token. */ public boolean isReserved(Token token) { return (token.getType() == TokenType.NAME) && isReserved(token.getString()); }