Exemplo n.º 1
0
  public static void main(String args[]) {
    Token tok;
    try {

      // Read from the command line for the filename
      Lexer lex = new Lexer(args[0]);

      while (true) {
        tok = lex.nextToken();

        // if token returns error token we print the line for error and exit
        if (tok.getKind() == Tokens.Error) {
          System.out.println(
              "Error in Line " + lex.source.getLineno() + " ErrorValue: " + tok.toString());
          System.exit(0);
        }
        // Printing the information about the line and the token
        String p =
            "Left: "
                + tok.getLeftPosition()
                + "| Right: "
                + tok.getRightPosition()
                + "| Type: "
                + TokenType.tokens.get(tok.getKind())
                + "| Value: ";
        // if ((tok.getKind() == Tokens.Identifier) || (tok.getKind() == Tokens.INTeger))
        p += tok.toString();
        System.out.println(p + "| Line: " + lex.source.getLineno());
      }
    } catch (Exception e) {
    }
  }
 public String toString(Token t) {
   if (t.getKind().hasPayload()) {
     return t.stringValue();
   } else {
     return t.kind.toString().toLowerCase();
   }
 }
 /**
  * Eat an identifier, possibly qualified (meaning that it is dotted). TODO AndyC Could create
  * complete identifiers (a.b.c) here rather than a sequence of them? (a, b, c)
  */
 private SpelNodeImpl eatPossiblyQualifiedId() {
   LinkedList<SpelNodeImpl> qualifiedIdPieces = new LinkedList<SpelNodeImpl>();
   Token node = peekToken();
   while (isValidQualifiedId(node)) {
     nextToken();
     if (node.kind != TokenKind.DOT) {
       qualifiedIdPieces.add(new Identifier(node.stringValue(), toPos(node)));
     }
     node = peekToken();
   }
   if (qualifiedIdPieces.isEmpty()) {
     if (node == null) {
       raiseInternalException(expressionString.length(), SpelMessage.OOD);
     }
     raiseInternalException(
         node.startpos,
         SpelMessage.NOT_EXPECTED_TOKEN,
         "qualified ID",
         node.getKind().toString().toLowerCase());
   }
   int pos =
       toPos(
           qualifiedIdPieces.getFirst().getStartPosition(),
           qualifiedIdPieces.getLast().getEndPosition());
   return new QualifiedIdentifier(
       pos, qualifiedIdPieces.toArray(new SpelNodeImpl[qualifiedIdPieces.size()]));
 }
Exemplo n.º 4
0
 public void process(JCas jCas, ResultSpecification rs) throws AnnotatorProcessException {
   JFSIndexRepository indexRep = jCas.getJFSIndexRepository();
   FSIndex tokenIndex = indexRep.getAnnotationIndex(Token.type);
   Iterator tokens = tokenIndex.iterator();
   while (tokens.hasNext()) {
     Token tok = (Token) tokens.next();
     System.out.print("Token: String=\"" + tok.getString() + "\", ");
     System.out.print("Kind=\"" + tok.getKind() + "\", ");
     System.out.println("Orth=\"" + tok.getOrth() + "\"");
   }
 }
 private Token eatToken(TokenKind expectedKind) {
   Token t = nextToken();
   if (t == null) {
     raiseInternalException(expressionString.length(), SpelMessage.OOD);
   }
   if (t.kind != expectedKind) {
     raiseInternalException(
         t.startpos,
         SpelMessage.NOT_EXPECTED_TOKEN,
         expectedKind.toString().toLowerCase(),
         t.getKind().toString().toLowerCase());
   }
   return t;
 }
 // powerExpr  : unaryExpression (POWER^ unaryExpression)? (INC || DEC) ;
 private SpelNodeImpl eatPowerIncDecExpression() {
   SpelNodeImpl expr = eatUnaryExpression();
   if (peekToken(TokenKind.POWER)) {
     Token t = nextToken(); // consume POWER
     SpelNodeImpl rhExpr = eatUnaryExpression();
     checkRightOperand(t, rhExpr);
     return new OperatorPower(toPos(t), expr, rhExpr);
   } else if (expr != null && peekToken(TokenKind.INC, TokenKind.DEC)) {
     Token t = nextToken(); // consume INC/DEC
     if (t.getKind() == TokenKind.INC) {
       return new OpInc(toPos(t), true, expr);
     } else {
       return new OpDec(toPos(t), true, expr);
     }
   }
   return expr;
 }
 // unaryExpression: (PLUS^ | MINUS^ | BANG^ | INC^ | DEC^) unaryExpression | primaryExpression ;
 private SpelNodeImpl eatUnaryExpression() {
   if (peekToken(TokenKind.PLUS, TokenKind.MINUS, TokenKind.NOT)) {
     Token t = nextToken();
     SpelNodeImpl expr = eatUnaryExpression();
     if (t.kind == TokenKind.NOT) {
       return new OperatorNot(toPos(t), expr);
     } else if (t.kind == TokenKind.PLUS) {
       return new OpPlus(toPos(t), expr);
     } else {
       Assert.isTrue(t.kind == TokenKind.MINUS);
       return new OpMinus(toPos(t), expr);
     }
   } else if (peekToken(TokenKind.INC, TokenKind.DEC)) {
     Token t = nextToken();
     SpelNodeImpl expr = eatUnaryExpression();
     if (t.getKind() == TokenKind.INC) {
       return new OpInc(toPos(t), false, expr);
     } else {
       return new OpDec(toPos(t), false, expr);
     }
   } else {
     return eatPrimaryExpression();
   }
 }