Exemple #1
0
  private List<Regex> regexPartial(SourcePosition pos) throws JmoException {
    List<Regex> regexList = new ArrayList<Regex>();
    while (true) {
      Token t = buf.getToken(0);
      if (t == null) {
        break;
      }
      if (t.getType() == TokenType.LBRACE) {
        // { found
        buf.takeToken();
        Regex rx = varPartial();
        expect(TokenType.RBRACE);
        regexList.add(rx);
      } else {
        String str = this.readUntil(TokenType.OR, TokenType.LBRACE);
        Regex literal = new LiteralPartial(str, pos);
        regexList.add(literal);

        Token current = buf.findValid();
        if (current == null || current.getType() == TokenType.OR) {
          break;
        }
      }
    }
    return regexList;
  }
 public Token consumeType(Token.Type type) throws IOException, ParseException {
   Token result = nextToken();
   if (result.getType() != type)
     throw new ParseException(
         "Expected token `" + type + "', found `" + result.getType() + "'", result.getOffset());
   return result;
 }
  protected void parseFormalParameters(Token token, SymTabEntry routineId) throws Exception {
    // Parse the formal parameters if there is an opening left parenthesis.
    token = synchronize(LEFT_PAREN_SET);
    if (token.getType() == LEFT_PAREN) {
      token = nextToken(); // consume (
      TokenType tokenType1 = token.getType();

      ArrayList<SymTabEntry> parms = new ArrayList<SymTabEntry>();
      while (tokenType1 != RIGHT_PAREN) {

        parms.addAll(parseParmSublist(token, routineId));

        token = currentToken();

        tokenType1 = token.getType();
      }

      // This sync needs to check for Int or Char
      token = synchronize(PARAMETER_SET);

      // Closing right parenthesis.
      if (token.getType() == RIGHT_PAREN) {
        token = nextToken(); // consume )
      } else {
        errorHandler.flag(token, MISSING_RIGHT_PAREN, this);
      }

      routineId.setAttribute(ROUTINE_PARMS, parms);
    }
  }
  public void parse(Token token) throws Exception {
    token = synchronize(IDENTIFIER_SET);

    // 遍历由分号分割的一系列变量申明
    while (token.getType() == IDENTIFIER) {

      // 解析每一个子表标识申明,比如 a,b,c : int
      parseIdentifierSublist(token);

      token = currentToken();
      TokenType tokenType = token.getType();

      // 一个定义由分号结束
      if (tokenType == SEMICOLON) {
        while (token.getType() == SEMICOLON) {
          token = nextToken(); // consume the ;
        }
      } else if (NEXT_START_SET.contains(tokenType)) {
        // 没有分号便开始下一个了。
        errorHandler.flag(token, MISSING_SEMICOLON, this);
      }

      token = synchronize(IDENTIFIER_SET);
    }
  }
  /**
   * Parse a term.
   *
   * @param token the initial token.
   * @return the root of the generated parse subtree.
   * @throws Exception if an error occurred.
   */
  private ICodeNode parseTerm(Token token) throws Exception {
    // Parse a factor and make its node the root node.
    ICodeNode rootNode = parseFactor(token);

    token = currentToken();
    TokenType tokenType = token.getType();

    // Loop over multiplicative operators.
    while (MULT_OPS.contains(tokenType)) {

      // Create a new operator node and adopt the current tree
      // as its first child.
      ICodeNodeType nodeType = MULT_OPS_OPS_MAP.get(tokenType);
      ICodeNode opNode = ICodeFactory.createICodeNode(nodeType);
      opNode.addChild(rootNode);

      token = nextToken(); // consume the operator

      // Parse another factor.  The operator node adopts
      // the term's tree as its second child.
      opNode.addChild(parseFactor(token));

      // The operator node becomes the new root node.
      rootNode = opNode;

      token = currentToken();
      tokenType = token.getType();
    }

    return rootNode;
  }
  private void expect(String string, Token... tokens) {
    System.out.println("Tokenizing [" + string + "]");
    ExprLexer tokenizer = new ExprLexer(string);
    int expectedIndex = 0;
    while (!tokenizer.isEndOfInput()) {
      String expectedString;
      Token expected;
      if (expectedIndex < tokens.length) {
        expected = tokens[expectedIndex];
        expectedString = expected.toString();
      } else {
        expected = null;
        expectedString = "<NOTHING>";
      }
      Token actual = tokenizer.next();
      System.out.println(
          String.format("Expected: %15s, got %s", expectedString, actual.toString()));
      if (expected != null) {
        assertEquals("tokenStart", expected.getTokenStart(), actual.getTokenStart());
        assertEquals("text", expected.getString(), actual.getString());
        assertEquals("type", expected.getType(), actual.getType());

        if (!expected.equals(actual)) {
          System.err.println("Unexpected result!");
          throw new AssertionError();
        }
      }
      expectedIndex++;
    }
  }
Exemple #7
0
  protected NaleNode parseVariable() throws JmoException {
    VarNode varNode = varNode();
    while (true) {
      Token t = buf.findValid();
      if (t == null) {
        return varNode;
      }
      if (t.getType() == TokenType.LPAREN) { // function
        expectValid(TokenType.LPAREN);
        List<ReferNode> paras = funcParasNode();
        expectValid(TokenType.RPAREN);
        FunctionCallNode fncN =
            new FunctionCallNode(
                t.getPosition(), varNode.getParNode(), varNode.getVarName(), paras);
        expectLineEnd();
        return fncN;
      } else if (t.getType() == TokenType.DOT) {
        buf.takeToken();
        VarNode varsub = varNode();
        varNode = new VarNode(varNode.getPosition(), varNode, varsub.getVarName());
      } else {

        return varNode;
      }
    }
  }
 /**
  * additive ::= multiplicative ADD multiplicative additive ::= multiplicative SUB multiplicative
  */
 private Value additive() {
   Value res = mulpiplicative();
   while (token.getType() == TokenType.ADD || token.getType() == TokenType.SUB) {
     if (token.getType() == TokenType.ADD) {
       res.setValue(res.getValue() + mulpiplicative().getValue());
     } else if (token.getType() == TokenType.SUB) {
       res.setValue(res.getValue() - mulpiplicative().getValue());
     }
   }
   return res;
 }
 /** multiplicative ::= unary MUL unary multiplicative ::= unary DIV unary */
 private Value mulpiplicative() {
   Value res = unary();
   while (token.getType() == TokenType.MUL || token.getType() == TokenType.DIV) {
     if (token.getType() == TokenType.MUL) {
       res.setValue(res.getValue() * unary().getValue());
     } else if (token.getType() == TokenType.DIV) {
       res.setValue(res.getValue() / unary().getValue());
     }
   }
   return res;
 }
Exemple #10
0
 /** equality ::= relational EQ relational equality ::= relational NEQ relational */
 private Value equality() {
   Value res = relational();
   while (token.getType() == TokenType.EQ || token.getType() == TokenType.NEQ) {
     if (token.getType() == TokenType.EQ) {
       res.setBooleanValue(res.getValue() == relational().getValue());
     } else if (token.getType() == TokenType.NEQ) {
       res.setBooleanValue(res.getValue() != relational().getValue());
     }
   }
   return res;
 }
Exemple #11
0
  private NaleNode statement() throws JmoException {

    Token t = buf.findValid();
    checkType(t, null);
    if (t.getType() == TokenType.SHARP) {
      buf.takeToken();
      t = buf.findValid();
      checkType(t, TokenType.VARIABLE);

      NaleNode nn = parseVariable();
      if (nn instanceof VarNode) {
        buf.findValid();
        expect(TokenType.ASSIGN);

        t = buf.findValid();
        checkType(t, null);
        return this.assign((VarNode) nn, true);

      } else {
        throw new ParseException("expect $ or exp but get " + nn, t.getPosition());
      }

    } else {
      if (t.getType() == TokenType.DOLLAR) {
        buf.takeToken();
      }
      t = buf.findValid();
      checkType(t, null);
      if (t.getType() == TokenType.VARIABLE) {
        NaleNode nn = parseVariable();
        if (nn instanceof VarNode) {
          Token next = buf.findValid();
          if (next == null) {
            return new FunctionCallNode(
                t.getPosition(), (VarNode) nn, "print", new ArrayList<ReferNode>(0));
          } else {
            expect(TokenType.ASSIGN);

            t = buf.findValid();
            checkType(t, null);
            return this.assign((VarNode) nn, false);
          }
        } else if (nn instanceof FunctionCallNode) {
          return nn;
        } else {
          throw new ParseException("expect $ or exp but get " + nn, t.getPosition());
        }
      } else {
        throw new ParseException("expect variable, but get " + t, t.getPosition());
      }
    }
  }
Exemple #12
0
 /**
  * How should a token be displayed in an error message? The default is to display just the text,
  * but during development you might want to have a lot of information spit out. Override in that
  * case to use t.toString() (which, for CommonToken, dumps everything about the token). This is
  * better than forcing you to override a method in your token objects because you don't have to go
  * modify your lexer so that it creates a new Java type.
  */
 public String getTokenErrorDisplay(Token t) {
   String s = t.getText();
   if (s == null) {
     if (t.getType() == Token.EOF) {
       s = "<EOF>";
     } else {
       s = "<" + t.getType() + ">";
     }
   }
   s = s.replaceAll("\n", "\\\\n");
   s = s.replaceAll("\r", "\\\\r");
   s = s.replaceAll("\t", "\\\\t");
   return "'" + s + "'";
 }
  /**
   * Parse a simple expression.
   *
   * @param token the initial token.
   * @return the root of the generated parse subtree.
   * @throws Exception if an error occurred.
   */
  private ICodeNode parseSimpleExpression(Token token) throws Exception {
    TokenType signType = null; // type of leading sign (if any)

    // Look for a leading + or - sign.
    TokenType tokenType = token.getType();
    if ((tokenType == PLUS) || (tokenType == MINUS)) {
      signType = tokenType;
      token = nextToken(); // consume the + or -
    }

    // Parse a term and make the root of its tree the root node.
    ICodeNode rootNode = parseTerm(token);

    // Was there a leading - sign?
    if (signType == MINUS) {

      // Create a NEGATE node and adopt the current tree
      // as its child. The NEGATE node becomes the new root node.
      ICodeNode negateNode = ICodeFactory.createICodeNode(NEGATE);
      negateNode.addChild(rootNode);
      rootNode = negateNode;
    }

    token = currentToken();
    tokenType = token.getType();

    // Loop over additive operators.
    while (ADD_OPS.contains(tokenType)) {

      // Create a new operator node and adopt the current tree
      // as its first child.
      ICodeNodeType nodeType = ADD_OPS_OPS_MAP.get(tokenType);
      ICodeNode opNode = ICodeFactory.createICodeNode(nodeType);
      opNode.addChild(rootNode);

      token = nextToken(); // consume the operator

      // Parse another term.  The operator node adopts
      // the term's tree as its second child.
      opNode.addChild(parseTerm(token));

      // The operator node becomes the new root node.
      rootNode = opNode;

      token = currentToken();
      tokenType = token.getType();
    }

    return rootNode;
  }
Exemple #14
0
 /** expression ::= logicalOr EOS */
 private Value expression() {
   Value res = logicalOr();
   if (token.getType() != TokenType.EOS) {
     throw expected(TokenType.EOS);
   }
   return res;
 }
  /**
   * Parse an expression.
   *
   * @param token the initial token.
   * @return the root of the generated parse subtree.
   * @throws Exception if an error occurred.
   */
  private ICodeNode parseExpression(Token token) throws Exception {
    // Parse a simple expression and make the root of its tree
    // the root node.
    ICodeNode rootNode = parseSimpleExpression(token);

    token = currentToken();
    TokenType tokenType = token.getType();

    // Look for a relational operator.
    if (REL_OPS.contains(tokenType)) {

      // Create a new operator node and adopt the current tree
      // as its first child.
      ICodeNodeType nodeType = REL_OPS_MAP.get(tokenType);
      ICodeNode opNode = ICodeFactory.createICodeNode(nodeType);
      opNode.addChild(rootNode);

      token = nextToken(); // consume the operator

      // Parse the second simple expression.  The operator node adopts
      // the simple expression's tree as its second child.
      opNode.addChild(parseSimpleExpression(token));

      // The operator node becomes the new root node.
      rootNode = opNode;
    }

    return rootNode;
  }
  private SymTabEntry parseRoutineName(Token token, String dummyName) throws Exception {
    SymTabEntry routineId = null;

    // Parse the routine name identifier.
    if (token.getType() == IDENTIFIER) {
      String routineName = token.getText().toLowerCase();
      routineId = symTabStack.lookupLocal(routineName);

      // Not already defined locally: Enter into the local symbol table.
      if (routineId == null) {
        routineId = symTabStack.enterLocal(routineName);
        routineId.appendLineNumber(token.getLineNumber());
      }

      // If already defined, it should be a forward definition.
      else {
        routineId = null;
        errorHandler.flag(token, IDENTIFIER_REDEFINED, this);
      }

      token = nextToken(); // consume routine name identifier

    } else {
      errorHandler.flag(token, MISSING_IDENTIFIER, this);
    }

    // If necessary, create a dummy routine name symbol table entry.
    if (routineId == null) {
      routineId = symTabStack.enterLocal(dummyName);
    }

    return routineId;
  }
  public static void main(String args[]) {

    if (args.length != 1) {
      System.err.println("Usage: java OCamlLexerTester <input.ml>");
      System.exit(1);
    }

    OCamlLexer lexer = new OCamlLexer();

    ArrayList<Token> tokens = new ArrayList<Token>();

    try {

      FileReader reader = new FileReader(args[0]);

      lexer.tokenize(reader, tokens);

      for (Token t : tokens)
        System.out.println(
            t.getType() + " - " + t.getStart() + "..." + (t.getStart() + t.getLength()));

    } catch (IOException e) {
      System.err.println("IO Exception");
      e.printStackTrace();
    }
  }
Exemple #18
0
  /**
   * Convert an arithmetic expression in infix notation (e.g 1+2) to postfix notation (e.g 12+) Only
   * simple expressions not containing brackets are supported. <br>
   * A description of the algorithm used to accomplish this task can be found at:
   * http://scriptasylum.com/tutorials/infix_postfix/algorithms/infix-postfix/index.htm <br>
   * Basically the list of {@code Tokens} is being scanned from left to right, all operands
   * (numbers) are pushed onto a {@code Stack}. If the {@code Token} is an operator it is compared
   * to the operator on top of the {@code Stack}. If the operator on top of the {@code Stack} has a
   * higher or equal weight than the current operator, the top of the {@code Stack} is pushed to the
   * postfix list. Therefore the operators with higher weight are going to be executed first when
   * the expression is evaluated. This process continues until the top of the {@code Stack} has a
   * lower weight than the current operator or the {@code Stack} is empty. Then the current operator
   * is pushed onto the {@code Stack}.
   *
   * @param tokenizer The {@code Tokenizer} which will be converted to postfix. It must not be
   *     {@code null}.
   * @return The resulting {@code List} of {@code Token}s in postfix order.
   */
  private static List<Token> postfix(Tokenizer tokenizer) {
    Stack<Token> stack = new Stack<>();
    List<Token> postfix = new LinkedList<>();
    BiPredicate<Token, Token> hasHigherOrEqualWeight =
        (t1, t2) ->
            Operators.getOpWeight(t1.getOperator()) - Operators.getOpWeight(t2.getOperator()) >= 0;

    while (tokenizer.hasNext()) {
      Token t = tokenizer.next();
      switch (t.getType()) {
        case NUMBER:
          postfix.add(t);
          break;
        case OPERATOR:
          if (!stack.isEmpty() && hasHigherOrEqualWeight.test(stack.peek(), t)) {
            while (!stack.isEmpty() && hasHigherOrEqualWeight.test(stack.peek(), t)) {
              postfix.add(stack.pop());
            }
            stack.push(t);
          } else {
            stack.push(t);
          }
          break;
        case OPENING_BRACKET:
        case CLOSING_BRACKET:
          throw new IllegalArgumentException(
              "Cannot create postfix expression if the source contains brackets.");
      }
    }

    while (!stack.isEmpty()) postfix.add(stack.pop());

    return postfix;
  }
Exemple #19
0
 /**
  * Consume and return the {@linkplain #getCurrentToken current symbol}.
  *
  * <p>E.g., given the following input with {@code A} being the current lookahead symbol, this
  * function moves the cursor to {@code B} and returns {@code A}.
  *
  * <pre>
  *  A B
  *  ^
  * </pre>
  *
  * If the parser is not in error recovery mode, the consumed symbol is added to the parse tree
  * using {@link ParserRuleContext#addChild(Token)}, and {@link ParseTreeListener#visitTerminal} is
  * called on any parse listeners. If the parser <em>is</em> in error recovery mode, the consumed
  * symbol is added to the parse tree using {@link ParserRuleContext#addErrorNode(Token)}, and
  * {@link ParseTreeListener#visitErrorNode} is called on any parse listeners.
  */
 public Token consume() {
   Token o = getCurrentToken();
   if (o.getType() != EOF) {
     getInputStream().consume();
   }
   boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty();
   if (_buildParseTrees || hasListener) {
     if (_errHandler.inErrorRecoveryMode(this)) {
       ErrorNode node = _ctx.addErrorNode(o);
       if (_parseListeners != null) {
         for (ParseTreeListener listener : _parseListeners) {
           listener.visitErrorNode(node);
         }
       }
     } else {
       TerminalNode node = _ctx.addChild(o);
       if (_parseListeners != null) {
         for (ParseTreeListener listener : _parseListeners) {
           listener.visitTerminal(node);
         }
       }
     }
   }
   return o;
 }
  public List<TerminalNode> getTokens(int ttype) {
    if (children == null) {
      return Collections.emptyList();
    }

    List<TerminalNode> tokens = null;
    for (ParseTree o : children) {
      if (o instanceof TerminalNode) {
        TerminalNode tnode = (TerminalNode) o;
        Token symbol = tnode.getSymbol();
        if (symbol.getType() == ttype) {
          if (tokens == null) {
            tokens = new ArrayList<TerminalNode>();
          }
          tokens.add(tnode);
        }
      }
    }

    if (tokens == null) {
      return Collections.emptyList();
    }

    return tokens;
  }
Exemple #21
0
 /**
  * Evaluate an expression in postfix notation. The expression must not contain brackes.
  *
  * <p>The algorithm used to perform the evaluation can be found at: algorithm from
  * http://scriptasylum.com/tutorials/infix_postfix/algorithms/postfix-evaluation/index.htm
  *
  * <p>Basically the expression is iterated over from left to right, all number tokens ({@code
  * double}) are pushed onto a {@code Stack} and all operator tokens are evaluated using the
  * elements on the {@code Stack}. Note that the operators '+' and '-' are special as they can be
  * unary (as opposed to binary) operators, which means it is possible for them to only operate on
  * one operand.
  *
  * @param postfix The postfix expression to evaluate. It must not be {@code null} and it must not
  *     contain brackets.
  * @return The result of the evaluation as a {@code double}
  */
 private static double evaluatePostfix(List<Token> postfix) {
   Stack<Double> stack = new Stack<>();
   for (Token t : postfix) {
     switch (t.getType()) {
       case NUMBER:
         stack.push(t.getNumber());
         break;
       case OPERATOR:
         Double t1, t2;
         t2 = stack.pop();
         Double result = 0.0;
         switch (t.getOperator()) {
           case "+":
             if (stack.isEmpty()) result = +t2;
             else {
               t1 = stack.pop();
               result = t1 + t2;
             }
             break;
           case "-":
             if (stack.isEmpty()) result = -t2;
             else {
               t1 = stack.pop();
               result = t1 - t2;
             }
             break;
           case "*":
             t1 = stack.pop();
             result = t1 * t2;
             break;
           case "/":
             t1 = stack.pop();
             result = t1 / t2;
             break;
           case "%":
             t1 = stack.pop();
             result = t1 % t2;
             break;
           case "^":
             t1 = stack.pop();
             result = Math.pow(t1, t2);
             break;
           case "~":
             result = (double) ~(long) (double) t2;
             break;
           default:
             t1 = stack.pop();
             result = Operators.getOperatorFunction((OperatorToken) t).apply(t1, t2);
         }
         stack.push(result);
         break;
       case OPENING_BRACKET:
       case CLOSING_BRACKET:
         throw new IllegalArgumentException(
             "Cannot evaluate postfix expression if it contains brackets.");
     }
   }
   return stack.pop();
 }
Exemple #22
0
 /** logicalAnd ::= equality AND equality */
 private Value logicalAnd() {
   Value res = equality();
   while (token.getType() == TokenType.AND) {
     Value operand = equality();
     res.setBooleanValue(res.getBooleanValue() && operand.getBooleanValue());
   }
   return res;
 }
Exemple #23
0
 /** logicalOr ::= logicalAnd OR logicalAnd */
 private Value logicalOr() {
   Value res = logicalAnd();
   while (token.getType() == TokenType.OR) {
     Value operand = logicalAnd();
     res.setBooleanValue(res.getBooleanValue() || operand.getBooleanValue());
   }
   return res;
 }
Exemple #24
0
  private void testSingleValue(String statement, String expectedData, TokenType expectedTokenType) {
    ArrayList<Token> tokens = Lexer.lex(statement);
    Assert.assertEquals(1, tokens.size());

    Token actualToken = tokens.get(0);
    Assert.assertEquals(expectedData, actualToken.getData());
    Assert.assertEquals(expectedTokenType, actualToken.getType());
  }
Exemple #25
0
    /**
     * Process a token. Handles common token processing functionality then delegates to the
     * individual concrete handlers.
     *
     * @param ctx the current parse context
     * @throws InvalidQueryException if unable to process the token
     */
    public void handleToken(ParseContext ctx) throws InvalidQueryException {
      Token token = ctx.getTokens()[ctx.getCurrentTokensIndex()];
      if (!validate(ctx.getPreviousTokenType())) {
        throw new InvalidQueryException(
            "Unexpected token encountered in query string. Last Token Type="
                + ctx.getPreviousTokenType()
                + ", Current Token[type="
                + token.getType()
                + ", value='"
                + token.getValue()
                + "']");
      }
      ctx.setTokenType(token.getType());

      int idxIncrement = _handleToken(ctx);
      ctx.setCurrentTokensIndex(ctx.getCurrentTokensIndex() + idxIncrement);
    }
 public static void main(String[] args) throws Exception {
   ANTLRFileStream input = new ANTLRFileStream(args[0]);
   WigLexer lexer = new WigLexer(input);
   Token token;
   while ((token = lexer.nextToken()) != Token.EOF_TOKEN) {
     System.out.println("Token: " + token.getText() + "(" + token.getType() + ")");
   }
 }
 /**
  * Get an integer option. Given the name of the option find its associated integer value. If the
  * associated value is not an integer or is not in the table, then throw an exception of type
  * NumberFormatException.
  *
  * @param key The name of the option
  * @return The value associated with the key.
  */
 public int getIntegerOption(String key) throws NumberFormatException {
   Token t = (Token) options.get(key);
   if (t == null || t.getType() != ANTLRTokenTypes.INT) {
     throw new NumberFormatException();
   } else {
     return Integer.parseInt(t.getText());
   }
 }
Exemple #28
0
 public final TokenList cloneTokens() {
   TokenList result = new TokenList();
   for (Token t = fFirst; t != null; t = (Token) t.getNext()) {
     if (t.getType() != CPreprocessor.tSCOPE_MARKER) {
       result.append(t.clone());
     }
   }
   return result;
 }
Exemple #29
0
 public void lex() {
   try {
     tok = split.getToken();
     text = tok.getString();
     val = tok.getType();
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
 /** add n elements to buffer */
 protected void fetch(int n) {
   for (int i = 1; i <= n; i++) {
     Token t = tokenSource.nextToken();
     t.setTokenIndex(tokens.size());
     // System.out.println("adding "+t+" at index "+tokens.size());
     tokens.add(t);
     if (t.getType() == Token.EOF) break;
   }
 }