private List<Regex> regexPartial(SourcePosition pos) throws JmoException { List<Regex> regexList = new ArrayList<Regex>(); while (true) { Token t = buf.getToken(0); if (t == null) { break; } if (t.getType() == TokenType.LBRACE) { // { found buf.takeToken(); Regex rx = varPartial(); expect(TokenType.RBRACE); regexList.add(rx); } else { String str = this.readUntil(TokenType.OR, TokenType.LBRACE); Regex literal = new LiteralPartial(str, pos); regexList.add(literal); Token current = buf.findValid(); if (current == null || current.getType() == TokenType.OR) { break; } } } return regexList; }
public Token consumeType(Token.Type type) throws IOException, ParseException { Token result = nextToken(); if (result.getType() != type) throw new ParseException( "Expected token `" + type + "', found `" + result.getType() + "'", result.getOffset()); return result; }
protected void parseFormalParameters(Token token, SymTabEntry routineId) throws Exception { // Parse the formal parameters if there is an opening left parenthesis. token = synchronize(LEFT_PAREN_SET); if (token.getType() == LEFT_PAREN) { token = nextToken(); // consume ( TokenType tokenType1 = token.getType(); ArrayList<SymTabEntry> parms = new ArrayList<SymTabEntry>(); while (tokenType1 != RIGHT_PAREN) { parms.addAll(parseParmSublist(token, routineId)); token = currentToken(); tokenType1 = token.getType(); } // This sync needs to check for Int or Char token = synchronize(PARAMETER_SET); // Closing right parenthesis. if (token.getType() == RIGHT_PAREN) { token = nextToken(); // consume ) } else { errorHandler.flag(token, MISSING_RIGHT_PAREN, this); } routineId.setAttribute(ROUTINE_PARMS, parms); } }
public void parse(Token token) throws Exception { token = synchronize(IDENTIFIER_SET); // 遍历由分号分割的一系列变量申明 while (token.getType() == IDENTIFIER) { // 解析每一个子表标识申明,比如 a,b,c : int parseIdentifierSublist(token); token = currentToken(); TokenType tokenType = token.getType(); // 一个定义由分号结束 if (tokenType == SEMICOLON) { while (token.getType() == SEMICOLON) { token = nextToken(); // consume the ; } } else if (NEXT_START_SET.contains(tokenType)) { // 没有分号便开始下一个了。 errorHandler.flag(token, MISSING_SEMICOLON, this); } token = synchronize(IDENTIFIER_SET); } }
/** * Parse a term. * * @param token the initial token. * @return the root of the generated parse subtree. * @throws Exception if an error occurred. */ private ICodeNode parseTerm(Token token) throws Exception { // Parse a factor and make its node the root node. ICodeNode rootNode = parseFactor(token); token = currentToken(); TokenType tokenType = token.getType(); // Loop over multiplicative operators. while (MULT_OPS.contains(tokenType)) { // Create a new operator node and adopt the current tree // as its first child. ICodeNodeType nodeType = MULT_OPS_OPS_MAP.get(tokenType); ICodeNode opNode = ICodeFactory.createICodeNode(nodeType); opNode.addChild(rootNode); token = nextToken(); // consume the operator // Parse another factor. The operator node adopts // the term's tree as its second child. opNode.addChild(parseFactor(token)); // The operator node becomes the new root node. rootNode = opNode; token = currentToken(); tokenType = token.getType(); } return rootNode; }
private void expect(String string, Token... tokens) { System.out.println("Tokenizing [" + string + "]"); ExprLexer tokenizer = new ExprLexer(string); int expectedIndex = 0; while (!tokenizer.isEndOfInput()) { String expectedString; Token expected; if (expectedIndex < tokens.length) { expected = tokens[expectedIndex]; expectedString = expected.toString(); } else { expected = null; expectedString = "<NOTHING>"; } Token actual = tokenizer.next(); System.out.println( String.format("Expected: %15s, got %s", expectedString, actual.toString())); if (expected != null) { assertEquals("tokenStart", expected.getTokenStart(), actual.getTokenStart()); assertEquals("text", expected.getString(), actual.getString()); assertEquals("type", expected.getType(), actual.getType()); if (!expected.equals(actual)) { System.err.println("Unexpected result!"); throw new AssertionError(); } } expectedIndex++; } }
protected NaleNode parseVariable() throws JmoException { VarNode varNode = varNode(); while (true) { Token t = buf.findValid(); if (t == null) { return varNode; } if (t.getType() == TokenType.LPAREN) { // function expectValid(TokenType.LPAREN); List<ReferNode> paras = funcParasNode(); expectValid(TokenType.RPAREN); FunctionCallNode fncN = new FunctionCallNode( t.getPosition(), varNode.getParNode(), varNode.getVarName(), paras); expectLineEnd(); return fncN; } else if (t.getType() == TokenType.DOT) { buf.takeToken(); VarNode varsub = varNode(); varNode = new VarNode(varNode.getPosition(), varNode, varsub.getVarName()); } else { return varNode; } } }
/** * additive ::= multiplicative ADD multiplicative additive ::= multiplicative SUB multiplicative */ private Value additive() { Value res = mulpiplicative(); while (token.getType() == TokenType.ADD || token.getType() == TokenType.SUB) { if (token.getType() == TokenType.ADD) { res.setValue(res.getValue() + mulpiplicative().getValue()); } else if (token.getType() == TokenType.SUB) { res.setValue(res.getValue() - mulpiplicative().getValue()); } } return res; }
/** multiplicative ::= unary MUL unary multiplicative ::= unary DIV unary */ private Value mulpiplicative() { Value res = unary(); while (token.getType() == TokenType.MUL || token.getType() == TokenType.DIV) { if (token.getType() == TokenType.MUL) { res.setValue(res.getValue() * unary().getValue()); } else if (token.getType() == TokenType.DIV) { res.setValue(res.getValue() / unary().getValue()); } } return res; }
/** equality ::= relational EQ relational equality ::= relational NEQ relational */ private Value equality() { Value res = relational(); while (token.getType() == TokenType.EQ || token.getType() == TokenType.NEQ) { if (token.getType() == TokenType.EQ) { res.setBooleanValue(res.getValue() == relational().getValue()); } else if (token.getType() == TokenType.NEQ) { res.setBooleanValue(res.getValue() != relational().getValue()); } } return res; }
private NaleNode statement() throws JmoException { Token t = buf.findValid(); checkType(t, null); if (t.getType() == TokenType.SHARP) { buf.takeToken(); t = buf.findValid(); checkType(t, TokenType.VARIABLE); NaleNode nn = parseVariable(); if (nn instanceof VarNode) { buf.findValid(); expect(TokenType.ASSIGN); t = buf.findValid(); checkType(t, null); return this.assign((VarNode) nn, true); } else { throw new ParseException("expect $ or exp but get " + nn, t.getPosition()); } } else { if (t.getType() == TokenType.DOLLAR) { buf.takeToken(); } t = buf.findValid(); checkType(t, null); if (t.getType() == TokenType.VARIABLE) { NaleNode nn = parseVariable(); if (nn instanceof VarNode) { Token next = buf.findValid(); if (next == null) { return new FunctionCallNode( t.getPosition(), (VarNode) nn, "print", new ArrayList<ReferNode>(0)); } else { expect(TokenType.ASSIGN); t = buf.findValid(); checkType(t, null); return this.assign((VarNode) nn, false); } } else if (nn instanceof FunctionCallNode) { return nn; } else { throw new ParseException("expect $ or exp but get " + nn, t.getPosition()); } } else { throw new ParseException("expect variable, but get " + t, t.getPosition()); } } }
/** * How should a token be displayed in an error message? The default is to display just the text, * but during development you might want to have a lot of information spit out. Override in that * case to use t.toString() (which, for CommonToken, dumps everything about the token). This is * better than forcing you to override a method in your token objects because you don't have to go * modify your lexer so that it creates a new Java type. */ public String getTokenErrorDisplay(Token t) { String s = t.getText(); if (s == null) { if (t.getType() == Token.EOF) { s = "<EOF>"; } else { s = "<" + t.getType() + ">"; } } s = s.replaceAll("\n", "\\\\n"); s = s.replaceAll("\r", "\\\\r"); s = s.replaceAll("\t", "\\\\t"); return "'" + s + "'"; }
/** * Parse a simple expression. * * @param token the initial token. * @return the root of the generated parse subtree. * @throws Exception if an error occurred. */ private ICodeNode parseSimpleExpression(Token token) throws Exception { TokenType signType = null; // type of leading sign (if any) // Look for a leading + or - sign. TokenType tokenType = token.getType(); if ((tokenType == PLUS) || (tokenType == MINUS)) { signType = tokenType; token = nextToken(); // consume the + or - } // Parse a term and make the root of its tree the root node. ICodeNode rootNode = parseTerm(token); // Was there a leading - sign? if (signType == MINUS) { // Create a NEGATE node and adopt the current tree // as its child. The NEGATE node becomes the new root node. ICodeNode negateNode = ICodeFactory.createICodeNode(NEGATE); negateNode.addChild(rootNode); rootNode = negateNode; } token = currentToken(); tokenType = token.getType(); // Loop over additive operators. while (ADD_OPS.contains(tokenType)) { // Create a new operator node and adopt the current tree // as its first child. ICodeNodeType nodeType = ADD_OPS_OPS_MAP.get(tokenType); ICodeNode opNode = ICodeFactory.createICodeNode(nodeType); opNode.addChild(rootNode); token = nextToken(); // consume the operator // Parse another term. The operator node adopts // the term's tree as its second child. opNode.addChild(parseTerm(token)); // The operator node becomes the new root node. rootNode = opNode; token = currentToken(); tokenType = token.getType(); } return rootNode; }
/** expression ::= logicalOr EOS */ private Value expression() { Value res = logicalOr(); if (token.getType() != TokenType.EOS) { throw expected(TokenType.EOS); } return res; }
/** * Parse an expression. * * @param token the initial token. * @return the root of the generated parse subtree. * @throws Exception if an error occurred. */ private ICodeNode parseExpression(Token token) throws Exception { // Parse a simple expression and make the root of its tree // the root node. ICodeNode rootNode = parseSimpleExpression(token); token = currentToken(); TokenType tokenType = token.getType(); // Look for a relational operator. if (REL_OPS.contains(tokenType)) { // Create a new operator node and adopt the current tree // as its first child. ICodeNodeType nodeType = REL_OPS_MAP.get(tokenType); ICodeNode opNode = ICodeFactory.createICodeNode(nodeType); opNode.addChild(rootNode); token = nextToken(); // consume the operator // Parse the second simple expression. The operator node adopts // the simple expression's tree as its second child. opNode.addChild(parseSimpleExpression(token)); // The operator node becomes the new root node. rootNode = opNode; } return rootNode; }
private SymTabEntry parseRoutineName(Token token, String dummyName) throws Exception { SymTabEntry routineId = null; // Parse the routine name identifier. if (token.getType() == IDENTIFIER) { String routineName = token.getText().toLowerCase(); routineId = symTabStack.lookupLocal(routineName); // Not already defined locally: Enter into the local symbol table. if (routineId == null) { routineId = symTabStack.enterLocal(routineName); routineId.appendLineNumber(token.getLineNumber()); } // If already defined, it should be a forward definition. else { routineId = null; errorHandler.flag(token, IDENTIFIER_REDEFINED, this); } token = nextToken(); // consume routine name identifier } else { errorHandler.flag(token, MISSING_IDENTIFIER, this); } // If necessary, create a dummy routine name symbol table entry. if (routineId == null) { routineId = symTabStack.enterLocal(dummyName); } return routineId; }
public static void main(String args[]) { if (args.length != 1) { System.err.println("Usage: java OCamlLexerTester <input.ml>"); System.exit(1); } OCamlLexer lexer = new OCamlLexer(); ArrayList<Token> tokens = new ArrayList<Token>(); try { FileReader reader = new FileReader(args[0]); lexer.tokenize(reader, tokens); for (Token t : tokens) System.out.println( t.getType() + " - " + t.getStart() + "..." + (t.getStart() + t.getLength())); } catch (IOException e) { System.err.println("IO Exception"); e.printStackTrace(); } }
/** * Convert an arithmetic expression in infix notation (e.g 1+2) to postfix notation (e.g 12+) Only * simple expressions not containing brackets are supported. <br> * A description of the algorithm used to accomplish this task can be found at: * http://scriptasylum.com/tutorials/infix_postfix/algorithms/infix-postfix/index.htm <br> * Basically the list of {@code Tokens} is being scanned from left to right, all operands * (numbers) are pushed onto a {@code Stack}. If the {@code Token} is an operator it is compared * to the operator on top of the {@code Stack}. If the operator on top of the {@code Stack} has a * higher or equal weight than the current operator, the top of the {@code Stack} is pushed to the * postfix list. Therefore the operators with higher weight are going to be executed first when * the expression is evaluated. This process continues until the top of the {@code Stack} has a * lower weight than the current operator or the {@code Stack} is empty. Then the current operator * is pushed onto the {@code Stack}. * * @param tokenizer The {@code Tokenizer} which will be converted to postfix. It must not be * {@code null}. * @return The resulting {@code List} of {@code Token}s in postfix order. */ private static List<Token> postfix(Tokenizer tokenizer) { Stack<Token> stack = new Stack<>(); List<Token> postfix = new LinkedList<>(); BiPredicate<Token, Token> hasHigherOrEqualWeight = (t1, t2) -> Operators.getOpWeight(t1.getOperator()) - Operators.getOpWeight(t2.getOperator()) >= 0; while (tokenizer.hasNext()) { Token t = tokenizer.next(); switch (t.getType()) { case NUMBER: postfix.add(t); break; case OPERATOR: if (!stack.isEmpty() && hasHigherOrEqualWeight.test(stack.peek(), t)) { while (!stack.isEmpty() && hasHigherOrEqualWeight.test(stack.peek(), t)) { postfix.add(stack.pop()); } stack.push(t); } else { stack.push(t); } break; case OPENING_BRACKET: case CLOSING_BRACKET: throw new IllegalArgumentException( "Cannot create postfix expression if the source contains brackets."); } } while (!stack.isEmpty()) postfix.add(stack.pop()); return postfix; }
/** * Consume and return the {@linkplain #getCurrentToken current symbol}. * * <p>E.g., given the following input with {@code A} being the current lookahead symbol, this * function moves the cursor to {@code B} and returns {@code A}. * * <pre> * A B * ^ * </pre> * * If the parser is not in error recovery mode, the consumed symbol is added to the parse tree * using {@link ParserRuleContext#addChild(Token)}, and {@link ParseTreeListener#visitTerminal} is * called on any parse listeners. If the parser <em>is</em> in error recovery mode, the consumed * symbol is added to the parse tree using {@link ParserRuleContext#addErrorNode(Token)}, and * {@link ParseTreeListener#visitErrorNode} is called on any parse listeners. */ public Token consume() { Token o = getCurrentToken(); if (o.getType() != EOF) { getInputStream().consume(); } boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty(); if (_buildParseTrees || hasListener) { if (_errHandler.inErrorRecoveryMode(this)) { ErrorNode node = _ctx.addErrorNode(o); if (_parseListeners != null) { for (ParseTreeListener listener : _parseListeners) { listener.visitErrorNode(node); } } } else { TerminalNode node = _ctx.addChild(o); if (_parseListeners != null) { for (ParseTreeListener listener : _parseListeners) { listener.visitTerminal(node); } } } } return o; }
public List<TerminalNode> getTokens(int ttype) { if (children == null) { return Collections.emptyList(); } List<TerminalNode> tokens = null; for (ParseTree o : children) { if (o instanceof TerminalNode) { TerminalNode tnode = (TerminalNode) o; Token symbol = tnode.getSymbol(); if (symbol.getType() == ttype) { if (tokens == null) { tokens = new ArrayList<TerminalNode>(); } tokens.add(tnode); } } } if (tokens == null) { return Collections.emptyList(); } return tokens; }
/** * Evaluate an expression in postfix notation. The expression must not contain brackes. * * <p>The algorithm used to perform the evaluation can be found at: algorithm from * http://scriptasylum.com/tutorials/infix_postfix/algorithms/postfix-evaluation/index.htm * * <p>Basically the expression is iterated over from left to right, all number tokens ({@code * double}) are pushed onto a {@code Stack} and all operator tokens are evaluated using the * elements on the {@code Stack}. Note that the operators '+' and '-' are special as they can be * unary (as opposed to binary) operators, which means it is possible for them to only operate on * one operand. * * @param postfix The postfix expression to evaluate. It must not be {@code null} and it must not * contain brackets. * @return The result of the evaluation as a {@code double} */ private static double evaluatePostfix(List<Token> postfix) { Stack<Double> stack = new Stack<>(); for (Token t : postfix) { switch (t.getType()) { case NUMBER: stack.push(t.getNumber()); break; case OPERATOR: Double t1, t2; t2 = stack.pop(); Double result = 0.0; switch (t.getOperator()) { case "+": if (stack.isEmpty()) result = +t2; else { t1 = stack.pop(); result = t1 + t2; } break; case "-": if (stack.isEmpty()) result = -t2; else { t1 = stack.pop(); result = t1 - t2; } break; case "*": t1 = stack.pop(); result = t1 * t2; break; case "/": t1 = stack.pop(); result = t1 / t2; break; case "%": t1 = stack.pop(); result = t1 % t2; break; case "^": t1 = stack.pop(); result = Math.pow(t1, t2); break; case "~": result = (double) ~(long) (double) t2; break; default: t1 = stack.pop(); result = Operators.getOperatorFunction((OperatorToken) t).apply(t1, t2); } stack.push(result); break; case OPENING_BRACKET: case CLOSING_BRACKET: throw new IllegalArgumentException( "Cannot evaluate postfix expression if it contains brackets."); } } return stack.pop(); }
/** logicalAnd ::= equality AND equality */ private Value logicalAnd() { Value res = equality(); while (token.getType() == TokenType.AND) { Value operand = equality(); res.setBooleanValue(res.getBooleanValue() && operand.getBooleanValue()); } return res; }
/** logicalOr ::= logicalAnd OR logicalAnd */ private Value logicalOr() { Value res = logicalAnd(); while (token.getType() == TokenType.OR) { Value operand = logicalAnd(); res.setBooleanValue(res.getBooleanValue() || operand.getBooleanValue()); } return res; }
private void testSingleValue(String statement, String expectedData, TokenType expectedTokenType) { ArrayList<Token> tokens = Lexer.lex(statement); Assert.assertEquals(1, tokens.size()); Token actualToken = tokens.get(0); Assert.assertEquals(expectedData, actualToken.getData()); Assert.assertEquals(expectedTokenType, actualToken.getType()); }
/** * Process a token. Handles common token processing functionality then delegates to the * individual concrete handlers. * * @param ctx the current parse context * @throws InvalidQueryException if unable to process the token */ public void handleToken(ParseContext ctx) throws InvalidQueryException { Token token = ctx.getTokens()[ctx.getCurrentTokensIndex()]; if (!validate(ctx.getPreviousTokenType())) { throw new InvalidQueryException( "Unexpected token encountered in query string. Last Token Type=" + ctx.getPreviousTokenType() + ", Current Token[type=" + token.getType() + ", value='" + token.getValue() + "']"); } ctx.setTokenType(token.getType()); int idxIncrement = _handleToken(ctx); ctx.setCurrentTokensIndex(ctx.getCurrentTokensIndex() + idxIncrement); }
public static void main(String[] args) throws Exception { ANTLRFileStream input = new ANTLRFileStream(args[0]); WigLexer lexer = new WigLexer(input); Token token; while ((token = lexer.nextToken()) != Token.EOF_TOKEN) { System.out.println("Token: " + token.getText() + "(" + token.getType() + ")"); } }
/** * Get an integer option. Given the name of the option find its associated integer value. If the * associated value is not an integer or is not in the table, then throw an exception of type * NumberFormatException. * * @param key The name of the option * @return The value associated with the key. */ public int getIntegerOption(String key) throws NumberFormatException { Token t = (Token) options.get(key); if (t == null || t.getType() != ANTLRTokenTypes.INT) { throw new NumberFormatException(); } else { return Integer.parseInt(t.getText()); } }
public final TokenList cloneTokens() { TokenList result = new TokenList(); for (Token t = fFirst; t != null; t = (Token) t.getNext()) { if (t.getType() != CPreprocessor.tSCOPE_MARKER) { result.append(t.clone()); } } return result; }
public void lex() { try { tok = split.getToken(); text = tok.getString(); val = tok.getType(); } catch (Exception e) { e.printStackTrace(); } }
/** add n elements to buffer */ protected void fetch(int n) { for (int i = 1; i <= n; i++) { Token t = tokenSource.nextToken(); t.setTokenIndex(tokens.size()); // System.out.println("adding "+t+" at index "+tokens.size()); tokens.add(t); if (t.getType() == Token.EOF) break; } }