public static void main(String args[]) {
Token tok;
try {
// Read from the command line for the filename
Lexer lex = new Lexer(args[0]);
while (true) {
tok = lex.nextToken();
// if token returns error token we print the line for error and exit
if (tok.getKind() == Tokens.Error) {
System.out.println(
"Error in Line " + lex.source.getLineno() + " ErrorValue: " + tok.toString());
System.exit(0);
}
// Printing the information about the line and the token
String p =
"Left: "
+ tok.getLeftPosition()
+ "| Right: "
+ tok.getRightPosition()
+ "| Type: "
+ TokenType.tokens.get(tok.getKind())
+ "| Value: ";
// if ((tok.getKind() == Tokens.Identifier) || (tok.getKind() == Tokens.INTeger))
p += tok.toString();
System.out.println(p + "| Line: " + lex.source.getLineno());
}
} catch (Exception e) {
}
}
public String toString(Token t) {
if (t.getKind().hasPayload()) {
return t.stringValue();
} else {
return t.kind.toString().toLowerCase();
}
}
/**
* Eat an identifier, possibly qualified (meaning that it is dotted). TODO AndyC Could create
* complete identifiers (a.b.c) here rather than a sequence of them? (a, b, c)
*/
private SpelNodeImpl eatPossiblyQualifiedId() {
LinkedList<SpelNodeImpl> qualifiedIdPieces = new LinkedList<SpelNodeImpl>();
Token node = peekToken();
while (isValidQualifiedId(node)) {
nextToken();
if (node.kind != TokenKind.DOT) {
qualifiedIdPieces.add(new Identifier(node.stringValue(), toPos(node)));
}
node = peekToken();
}
if (qualifiedIdPieces.isEmpty()) {
if (node == null) {
raiseInternalException(expressionString.length(), SpelMessage.OOD);
}
raiseInternalException(
node.startpos,
SpelMessage.NOT_EXPECTED_TOKEN,
"qualified ID",
node.getKind().toString().toLowerCase());
}
int pos =
toPos(
qualifiedIdPieces.getFirst().getStartPosition(),
qualifiedIdPieces.getLast().getEndPosition());
return new QualifiedIdentifier(
pos, qualifiedIdPieces.toArray(new SpelNodeImpl[qualifiedIdPieces.size()]));
}
public void process(JCas jCas, ResultSpecification rs) throws AnnotatorProcessException {
JFSIndexRepository indexRep = jCas.getJFSIndexRepository();
FSIndex tokenIndex = indexRep.getAnnotationIndex(Token.type);
Iterator tokens = tokenIndex.iterator();
while (tokens.hasNext()) {
Token tok = (Token) tokens.next();
System.out.print("Token: String=\"" + tok.getString() + "\", ");
System.out.print("Kind=\"" + tok.getKind() + "\", ");
System.out.println("Orth=\"" + tok.getOrth() + "\"");
}
}
private Token eatToken(TokenKind expectedKind) {
Token t = nextToken();
if (t == null) {
raiseInternalException(expressionString.length(), SpelMessage.OOD);
}
if (t.kind != expectedKind) {
raiseInternalException(
t.startpos,
SpelMessage.NOT_EXPECTED_TOKEN,
expectedKind.toString().toLowerCase(),
t.getKind().toString().toLowerCase());
}
return t;
}
// powerExpr : unaryExpression (POWER^ unaryExpression)? (INC || DEC) ;
private SpelNodeImpl eatPowerIncDecExpression() {
SpelNodeImpl expr = eatUnaryExpression();
if (peekToken(TokenKind.POWER)) {
Token t = nextToken(); // consume POWER
SpelNodeImpl rhExpr = eatUnaryExpression();
checkRightOperand(t, rhExpr);
return new OperatorPower(toPos(t), expr, rhExpr);
} else if (expr != null && peekToken(TokenKind.INC, TokenKind.DEC)) {
Token t = nextToken(); // consume INC/DEC
if (t.getKind() == TokenKind.INC) {
return new OpInc(toPos(t), true, expr);
} else {
return new OpDec(toPos(t), true, expr);
}
}
return expr;
}
// unaryExpression: (PLUS^ | MINUS^ | BANG^ | INC^ | DEC^) unaryExpression | primaryExpression ;
private SpelNodeImpl eatUnaryExpression() {
if (peekToken(TokenKind.PLUS, TokenKind.MINUS, TokenKind.NOT)) {
Token t = nextToken();
SpelNodeImpl expr = eatUnaryExpression();
if (t.kind == TokenKind.NOT) {
return new OperatorNot(toPos(t), expr);
} else if (t.kind == TokenKind.PLUS) {
return new OpPlus(toPos(t), expr);
} else {
Assert.isTrue(t.kind == TokenKind.MINUS);
return new OpMinus(toPos(t), expr);
}
} else if (peekToken(TokenKind.INC, TokenKind.DEC)) {
Token t = nextToken();
SpelNodeImpl expr = eatUnaryExpression();
if (t.getKind() == TokenKind.INC) {
return new OpInc(toPos(t), false, expr);
} else {
return new OpDec(toPos(t), false, expr);
}
} else {
return eatPrimaryExpression();
}
}