public static void main(String args[]) { Token tok; try { // Read from the command line for the filename Lexer lex = new Lexer(args[0]); while (true) { tok = lex.nextToken(); // if token returns error token we print the line for error and exit if (tok.getKind() == Tokens.Error) { System.out.println( "Error in Line " + lex.source.getLineno() + " ErrorValue: " + tok.toString()); System.exit(0); } // Printing the information about the line and the token String p = "Left: " + tok.getLeftPosition() + "| Right: " + tok.getRightPosition() + "| Type: " + TokenType.tokens.get(tok.getKind()) + "| Value: "; // if ((tok.getKind() == Tokens.Identifier) || (tok.getKind() == Tokens.INTeger)) p += tok.toString(); System.out.println(p + "| Line: " + lex.source.getLineno()); } } catch (Exception e) { } }
public String toString(Token t) { if (t.getKind().hasPayload()) { return t.stringValue(); } else { return t.kind.toString().toLowerCase(); } }
/** * Eat an identifier, possibly qualified (meaning that it is dotted). TODO AndyC Could create * complete identifiers (a.b.c) here rather than a sequence of them? (a, b, c) */ private SpelNodeImpl eatPossiblyQualifiedId() { LinkedList<SpelNodeImpl> qualifiedIdPieces = new LinkedList<SpelNodeImpl>(); Token node = peekToken(); while (isValidQualifiedId(node)) { nextToken(); if (node.kind != TokenKind.DOT) { qualifiedIdPieces.add(new Identifier(node.stringValue(), toPos(node))); } node = peekToken(); } if (qualifiedIdPieces.isEmpty()) { if (node == null) { raiseInternalException(expressionString.length(), SpelMessage.OOD); } raiseInternalException( node.startpos, SpelMessage.NOT_EXPECTED_TOKEN, "qualified ID", node.getKind().toString().toLowerCase()); } int pos = toPos( qualifiedIdPieces.getFirst().getStartPosition(), qualifiedIdPieces.getLast().getEndPosition()); return new QualifiedIdentifier( pos, qualifiedIdPieces.toArray(new SpelNodeImpl[qualifiedIdPieces.size()])); }
public void process(JCas jCas, ResultSpecification rs) throws AnnotatorProcessException { JFSIndexRepository indexRep = jCas.getJFSIndexRepository(); FSIndex tokenIndex = indexRep.getAnnotationIndex(Token.type); Iterator tokens = tokenIndex.iterator(); while (tokens.hasNext()) { Token tok = (Token) tokens.next(); System.out.print("Token: String=\"" + tok.getString() + "\", "); System.out.print("Kind=\"" + tok.getKind() + "\", "); System.out.println("Orth=\"" + tok.getOrth() + "\""); } }
private Token eatToken(TokenKind expectedKind) { Token t = nextToken(); if (t == null) { raiseInternalException(expressionString.length(), SpelMessage.OOD); } if (t.kind != expectedKind) { raiseInternalException( t.startpos, SpelMessage.NOT_EXPECTED_TOKEN, expectedKind.toString().toLowerCase(), t.getKind().toString().toLowerCase()); } return t; }
// powerExpr : unaryExpression (POWER^ unaryExpression)? (INC || DEC) ; private SpelNodeImpl eatPowerIncDecExpression() { SpelNodeImpl expr = eatUnaryExpression(); if (peekToken(TokenKind.POWER)) { Token t = nextToken(); // consume POWER SpelNodeImpl rhExpr = eatUnaryExpression(); checkRightOperand(t, rhExpr); return new OperatorPower(toPos(t), expr, rhExpr); } else if (expr != null && peekToken(TokenKind.INC, TokenKind.DEC)) { Token t = nextToken(); // consume INC/DEC if (t.getKind() == TokenKind.INC) { return new OpInc(toPos(t), true, expr); } else { return new OpDec(toPos(t), true, expr); } } return expr; }
// unaryExpression: (PLUS^ | MINUS^ | BANG^ | INC^ | DEC^) unaryExpression | primaryExpression ; private SpelNodeImpl eatUnaryExpression() { if (peekToken(TokenKind.PLUS, TokenKind.MINUS, TokenKind.NOT)) { Token t = nextToken(); SpelNodeImpl expr = eatUnaryExpression(); if (t.kind == TokenKind.NOT) { return new OperatorNot(toPos(t), expr); } else if (t.kind == TokenKind.PLUS) { return new OpPlus(toPos(t), expr); } else { Assert.isTrue(t.kind == TokenKind.MINUS); return new OpMinus(toPos(t), expr); } } else if (peekToken(TokenKind.INC, TokenKind.DEC)) { Token t = nextToken(); SpelNodeImpl expr = eatUnaryExpression(); if (t.getKind() == TokenKind.INC) { return new OpInc(toPos(t), false, expr); } else { return new OpDec(toPos(t), false, expr); } } else { return eatPrimaryExpression(); } }