private void expect(String string, Token... tokens) {
System.out.println("Tokenizing [" + string + "]");
ExprLexer tokenizer = new ExprLexer(string);
int expectedIndex = 0;
while (!tokenizer.isEndOfInput()) {
String expectedString;
Token expected;
if (expectedIndex < tokens.length) {
expected = tokens[expectedIndex];
expectedString = expected.toString();
} else {
expected = null;
expectedString = "<NOTHING>";
}
Token actual = tokenizer.next();
System.out.println(
String.format("Expected: %15s, got %s", expectedString, actual.toString()));
if (expected != null) {
assertEquals("tokenStart", expected.getTokenStart(), actual.getTokenStart());
assertEquals("text", expected.getString(), actual.getString());
assertEquals("type", expected.getType(), actual.getType());
if (!expected.equals(actual)) {
System.err.println("Unexpected result!");
throw new AssertionError();
}
}
expectedIndex++;
}
}
private boolean isSingeLineComment() {
if (m_CurrChar.getString().equals("/") && m_NextChar.getString().equals("/")) {
m_InComment = true;
return true;
}
return false;
}
public Token next() {
try {
if (m_NextNext.isEmpty() == false) {
Token next = m_NextNext.get(0);
m_NextNext.remove(0);
m_Ret = new StringBuilder();
return next;
}
if (m_CurrChar.isImmutable()) {
Token ret1 = m_CurrChar;
read();
m_Ret = new StringBuilder();
return ret1;
}
while (shouldTrim(m_CurrChar.getString())) read();
if (m_CurrChar.isImmutable()) {
Token ret1 = m_CurrChar;
read();
m_Ret = new StringBuilder();
return ret1;
}
Token ret = doNext();
while (shouldTrim(m_CurrChar.getString())) read();
m_Ret = new StringBuilder();
return ret;
} catch (Exception ex) {
m_EndOfStream = true;
if (m_Ret == null) m_Ret = new StringBuilder();
String ret_string = m_Ret.toString();
Token ret = new Token(ret_string.trim());
return ret;
}
}
private boolean isSpecialEndToken() {
String curr;
if (m_SpecialEndToken.length() == 2) {
curr = m_CurrChar.getString() + m_NextChar.getString();
} else {
curr = m_CurrChar.getString();
}
if (curr.equals(m_SpecialEndToken)) {
if (m_SpecialEndToken.equals("*/")) {
m_InComment = false;
}
return true;
}
return false;
}
private boolean isSpecialStartToken() {
String two = m_CurrChar.getString() + m_NextChar.getString();
if (two.equals("/*")) {
m_InComment = true;
m_SpecialEndToken = "*/";
return true;
} else if (m_CurrChar.getString().equals("\"") && m_InComment == false) {
m_SpecialEndToken = "\"";
return true;
} else if (m_CurrChar.getString().equals("'") && m_InComment == false) {
m_SpecialEndToken = "\'";
return true;
}
return false;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
for (Token t : tokens)
sb.append(t.toString()).append(" String:").append(t.getString()).append("\n");
return sb.toString();
}
public void lex() {
try {
tok = split.getToken();
text = tok.getString();
val = tok.getType();
} catch (Exception e) {
e.printStackTrace();
}
}
public void process(JCas jCas, ResultSpecification rs) throws AnnotatorProcessException {
JFSIndexRepository indexRep = jCas.getJFSIndexRepository();
FSIndex tokenIndex = indexRep.getAnnotationIndex(Token.type);
Iterator tokens = tokenIndex.iterator();
while (tokens.hasNext()) {
Token tok = (Token) tokens.next();
System.out.print("Token: String=\"" + tok.getString() + "\", ");
System.out.print("Kind=\"" + tok.getKind() + "\", ");
System.out.println("Orth=\"" + tok.getOrth() + "\"");
}
}
private Token readSpecial() throws Exception {
String ret = null;
try {
ret = m_CurrChar.getString();
read();
while (isSpecialEndToken() == false) {
ret += m_CurrChar.getString();
read();
}
read();
if (m_SpecialEndToken.length() == 2) read();
ret += m_SpecialEndToken;
Token token = new Token(ret);
token.setImmutable();
return token;
} catch (Exception ex) {
Token token = new Token(ret);
token.setImmutable();
return token;
}
}
private Token eatLine() throws Exception {
m_Ret = new StringBuilder();
m_Ret.append(m_CurrChar.getString());
List<Token> all_ret = new ArrayList<Token>();
boolean is_immutable = false;
if (isSingeLineComment()) is_immutable = true;
while (true) {
read();
if (isSpecialStartToken()) {
Token curr = new Token(m_Ret.toString());
if (is_immutable) curr.setImmutable();
all_ret.add(curr);
all_ret.add(readSpecial());
is_immutable = false;
m_Ret = new StringBuilder();
} else if (m_InComment == false && isSingeLineComment()) {
Token curr = new Token(m_Ret.toString());
is_immutable = true;
all_ret.add(curr);
m_Ret = new StringBuilder();
} else if (isNewLine(m_CurrChar.getString())) {
Token curr = new Token(m_Ret.toString());
if (is_immutable) curr.setImmutable();
all_ret.add(curr);
Token newline = new Token("\n");
newline.setNewLine();
all_ret.add(newline);
Token ret = all_ret.get(0);
all_ret.remove(0);
m_NextNext.addAll(all_ret);
is_immutable = false;
// ignore # lines like they are comments
// ret.setComment();
return ret;
}
m_Ret.append(m_CurrChar.getString());
}
}
@Override
public Expr parse(MagpieParser parser, Token token) {
switch (token.getType()) {
case BOOL:
return Expr.bool(token.getPosition(), token.getBool());
case INT:
return Expr.int_(token.getPosition(), token.getInt());
case NOTHING:
return Expr.nothing(token.getPosition());
case STRING:
return Expr.string(token.getPosition(), token.getString());
}
throw new ParseException("Unexpected token type for literal.");
}
private Token doNext() throws Exception {
if (isSingleLineToken()) {
Token token = eatLine();
m_InComment = false;
token.setImmutable();
return token;
}
if (isSpecialStartToken()) {
return readSpecial();
} else {
m_Ret = new StringBuilder();
while (isSpecialStartToken() == false && isSingleLineToken() == false) {
m_Ret.append(m_CurrChar.getString());
read();
}
Token ret = new Token(m_Ret.toString());
return ret;
}
}
public int getPrecedence(Token token) {
int precedence = 0;
// A reserved word can't start an infix expression. Prevents us from
// parsing a keyword as an identifier.
if (isReserved(token)) return 0;
// If we have a prefix parser for this token's name, then that takes
// precedence. Prevents us from parsing a keyword as an identifier.
if ((token.getValue() instanceof String) && mPrefixParsers.isReserved(token.getString()))
return 0;
InfixParser parser = mInfixParsers.get(token);
if (parser != null) {
precedence = parser.getPrecedence();
}
return precedence;
}
private boolean isSingleLineToken() {
if (m_CurrChar.getString().equals("#")) return true;
if (isSingeLineComment()) return true;
return false;
}
/**
* Gets whether or not this token is a reserved word. Reserved words like "else" and "then" are
* claimed for special use by mixfix parsers, so can't be parsed on their own.
*
* @param token The token to test
* @return True if the token is a reserved name token.
*/
public boolean isReserved(Token token) {
return (token.getType() == TokenType.NAME) && isReserved(token.getString());
}