private List<Regex> regexPartial(SourcePosition pos) throws JmoException {
List<Regex> regexList = new ArrayList<Regex>();
while (true) {
Token t = buf.getToken(0);
if (t == null) {
break;
}
if (t.getType() == TokenType.LBRACE) {
// { found
buf.takeToken();
Regex rx = varPartial();
expect(TokenType.RBRACE);
regexList.add(rx);
} else {
String str = this.readUntil(TokenType.OR, TokenType.LBRACE);
Regex literal = new LiteralPartial(str, pos);
regexList.add(literal);
Token current = buf.findValid();
if (current == null || current.getType() == TokenType.OR) {
break;
}
}
}
return regexList;
}
public Token consumeType(Token.Type type) throws IOException, ParseException {
Token result = nextToken();
if (result.getType() != type)
throw new ParseException(
"Expected token `" + type + "', found `" + result.getType() + "'", result.getOffset());
return result;
}
protected void parseFormalParameters(Token token, SymTabEntry routineId) throws Exception {
// Parse the formal parameters if there is an opening left parenthesis.
token = synchronize(LEFT_PAREN_SET);
if (token.getType() == LEFT_PAREN) {
token = nextToken(); // consume (
TokenType tokenType1 = token.getType();
ArrayList<SymTabEntry> parms = new ArrayList<SymTabEntry>();
while (tokenType1 != RIGHT_PAREN) {
parms.addAll(parseParmSublist(token, routineId));
token = currentToken();
tokenType1 = token.getType();
}
// This sync needs to check for Int or Char
token = synchronize(PARAMETER_SET);
// Closing right parenthesis.
if (token.getType() == RIGHT_PAREN) {
token = nextToken(); // consume )
} else {
errorHandler.flag(token, MISSING_RIGHT_PAREN, this);
}
routineId.setAttribute(ROUTINE_PARMS, parms);
}
}
public void parse(Token token) throws Exception {
token = synchronize(IDENTIFIER_SET);
// 遍历由分号分割的一系列变量申明
while (token.getType() == IDENTIFIER) {
// 解析每一个子表标识申明，比如 a,b,c : int
parseIdentifierSublist(token);
token = currentToken();
TokenType tokenType = token.getType();
// 一个定义由分号结束
if (tokenType == SEMICOLON) {
while (token.getType() == SEMICOLON) {
token = nextToken(); // consume the ;
}
} else if (NEXT_START_SET.contains(tokenType)) {
// 没有分号便开始下一个了。
errorHandler.flag(token, MISSING_SEMICOLON, this);
}
token = synchronize(IDENTIFIER_SET);
}
}
/**
* Parse a term.
*
* @param token the initial token.
* @return the root of the generated parse subtree.
* @throws Exception if an error occurred.
*/
private ICodeNode parseTerm(Token token) throws Exception {
// Parse a factor and make its node the root node.
ICodeNode rootNode = parseFactor(token);
token = currentToken();
TokenType tokenType = token.getType();
// Loop over multiplicative operators.
while (MULT_OPS.contains(tokenType)) {
// Create a new operator node and adopt the current tree
// as its first child.
ICodeNodeType nodeType = MULT_OPS_OPS_MAP.get(tokenType);
ICodeNode opNode = ICodeFactory.createICodeNode(nodeType);
opNode.addChild(rootNode);
token = nextToken(); // consume the operator
// Parse another factor. The operator node adopts
// the term's tree as its second child.
opNode.addChild(parseFactor(token));
// The operator node becomes the new root node.
rootNode = opNode;
token = currentToken();
tokenType = token.getType();
}
return rootNode;
}
private void expect(String string, Token... tokens) {
System.out.println("Tokenizing [" + string + "]");
ExprLexer tokenizer = new ExprLexer(string);
int expectedIndex = 0;
while (!tokenizer.isEndOfInput()) {
String expectedString;
Token expected;
if (expectedIndex < tokens.length) {
expected = tokens[expectedIndex];
expectedString = expected.toString();
} else {
expected = null;
expectedString = "<NOTHING>";
}
Token actual = tokenizer.next();
System.out.println(
String.format("Expected: %15s, got %s", expectedString, actual.toString()));
if (expected != null) {
assertEquals("tokenStart", expected.getTokenStart(), actual.getTokenStart());
assertEquals("text", expected.getString(), actual.getString());
assertEquals("type", expected.getType(), actual.getType());
if (!expected.equals(actual)) {
System.err.println("Unexpected result!");
throw new AssertionError();
}
}
expectedIndex++;
}
}
protected NaleNode parseVariable() throws JmoException {
VarNode varNode = varNode();
while (true) {
Token t = buf.findValid();
if (t == null) {
return varNode;
}
if (t.getType() == TokenType.LPAREN) { // function
expectValid(TokenType.LPAREN);
List<ReferNode> paras = funcParasNode();
expectValid(TokenType.RPAREN);
FunctionCallNode fncN =
new FunctionCallNode(
t.getPosition(), varNode.getParNode(), varNode.getVarName(), paras);
expectLineEnd();
return fncN;
} else if (t.getType() == TokenType.DOT) {
buf.takeToken();
VarNode varsub = varNode();
varNode = new VarNode(varNode.getPosition(), varNode, varsub.getVarName());
} else {
return varNode;
}
}
}
/**
* additive ::= multiplicative ADD multiplicative additive ::= multiplicative SUB multiplicative
*/
private Value additive() {
Value res = mulpiplicative();
while (token.getType() == TokenType.ADD || token.getType() == TokenType.SUB) {
if (token.getType() == TokenType.ADD) {
res.setValue(res.getValue() + mulpiplicative().getValue());
} else if (token.getType() == TokenType.SUB) {
res.setValue(res.getValue() - mulpiplicative().getValue());
}
}
return res;
}
/** multiplicative ::= unary MUL unary multiplicative ::= unary DIV unary */
private Value mulpiplicative() {
Value res = unary();
while (token.getType() == TokenType.MUL || token.getType() == TokenType.DIV) {
if (token.getType() == TokenType.MUL) {
res.setValue(res.getValue() * unary().getValue());
} else if (token.getType() == TokenType.DIV) {
res.setValue(res.getValue() / unary().getValue());
}
}
return res;
}
/** equality ::= relational EQ relational equality ::= relational NEQ relational */
private Value equality() {
Value res = relational();
while (token.getType() == TokenType.EQ || token.getType() == TokenType.NEQ) {
if (token.getType() == TokenType.EQ) {
res.setBooleanValue(res.getValue() == relational().getValue());
} else if (token.getType() == TokenType.NEQ) {
res.setBooleanValue(res.getValue() != relational().getValue());
}
}
return res;
}
private NaleNode statement() throws JmoException {
Token t = buf.findValid();
checkType(t, null);
if (t.getType() == TokenType.SHARP) {
buf.takeToken();
t = buf.findValid();
checkType(t, TokenType.VARIABLE);
NaleNode nn = parseVariable();
if (nn instanceof VarNode) {
buf.findValid();
expect(TokenType.ASSIGN);
t = buf.findValid();
checkType(t, null);
return this.assign((VarNode) nn, true);
} else {
throw new ParseException("expect $ or exp but get " + nn, t.getPosition());
}
} else {
if (t.getType() == TokenType.DOLLAR) {
buf.takeToken();
}
t = buf.findValid();
checkType(t, null);
if (t.getType() == TokenType.VARIABLE) {
NaleNode nn = parseVariable();
if (nn instanceof VarNode) {
Token next = buf.findValid();
if (next == null) {
return new FunctionCallNode(
t.getPosition(), (VarNode) nn, "print", new ArrayList<ReferNode>(0));
} else {
expect(TokenType.ASSIGN);
t = buf.findValid();
checkType(t, null);
return this.assign((VarNode) nn, false);
}
} else if (nn instanceof FunctionCallNode) {
return nn;
} else {
throw new ParseException("expect $ or exp but get " + nn, t.getPosition());
}
} else {
throw new ParseException("expect variable, but get " + t, t.getPosition());
}
}
}
/**
* How should a token be displayed in an error message? The default is to display just the text,
* but during development you might want to have a lot of information spit out. Override in that
* case to use t.toString() (which, for CommonToken, dumps everything about the token). This is
* better than forcing you to override a method in your token objects because you don't have to go
* modify your lexer so that it creates a new Java type.
*/
public String getTokenErrorDisplay(Token t) {
String s = t.getText();
if (s == null) {
if (t.getType() == Token.EOF) {
s = "<EOF>";
} else {
s = "<" + t.getType() + ">";
}
}
s = s.replaceAll("\n", "\\\\n");
s = s.replaceAll("\r", "\\\\r");
s = s.replaceAll("\t", "\\\\t");
return "'" + s + "'";
}
/**
* Parse a simple expression.
*
* @param token the initial token.
* @return the root of the generated parse subtree.
* @throws Exception if an error occurred.
*/
private ICodeNode parseSimpleExpression(Token token) throws Exception {
TokenType signType = null; // type of leading sign (if any)
// Look for a leading + or - sign.
TokenType tokenType = token.getType();
if ((tokenType == PLUS) || (tokenType == MINUS)) {
signType = tokenType;
token = nextToken(); // consume the + or -
}
// Parse a term and make the root of its tree the root node.
ICodeNode rootNode = parseTerm(token);
// Was there a leading - sign?
if (signType == MINUS) {
// Create a NEGATE node and adopt the current tree
// as its child. The NEGATE node becomes the new root node.
ICodeNode negateNode = ICodeFactory.createICodeNode(NEGATE);
negateNode.addChild(rootNode);
rootNode = negateNode;
}
token = currentToken();
tokenType = token.getType();
// Loop over additive operators.
while (ADD_OPS.contains(tokenType)) {
// Create a new operator node and adopt the current tree
// as its first child.
ICodeNodeType nodeType = ADD_OPS_OPS_MAP.get(tokenType);
ICodeNode opNode = ICodeFactory.createICodeNode(nodeType);
opNode.addChild(rootNode);
token = nextToken(); // consume the operator
// Parse another term. The operator node adopts
// the term's tree as its second child.
opNode.addChild(parseTerm(token));
// The operator node becomes the new root node.
rootNode = opNode;
token = currentToken();
tokenType = token.getType();
}
return rootNode;
}
/** expression ::= logicalOr EOS */
private Value expression() {
Value res = logicalOr();
if (token.getType() != TokenType.EOS) {
throw expected(TokenType.EOS);
}
return res;
}
/**
* Parse an expression.
*
* @param token the initial token.
* @return the root of the generated parse subtree.
* @throws Exception if an error occurred.
*/
private ICodeNode parseExpression(Token token) throws Exception {
// Parse a simple expression and make the root of its tree
// the root node.
ICodeNode rootNode = parseSimpleExpression(token);
token = currentToken();
TokenType tokenType = token.getType();
// Look for a relational operator.
if (REL_OPS.contains(tokenType)) {
// Create a new operator node and adopt the current tree
// as its first child.
ICodeNodeType nodeType = REL_OPS_MAP.get(tokenType);
ICodeNode opNode = ICodeFactory.createICodeNode(nodeType);
opNode.addChild(rootNode);
token = nextToken(); // consume the operator
// Parse the second simple expression. The operator node adopts
// the simple expression's tree as its second child.
opNode.addChild(parseSimpleExpression(token));
// The operator node becomes the new root node.
rootNode = opNode;
}
return rootNode;
}
private SymTabEntry parseRoutineName(Token token, String dummyName) throws Exception {
SymTabEntry routineId = null;
// Parse the routine name identifier.
if (token.getType() == IDENTIFIER) {
String routineName = token.getText().toLowerCase();
routineId = symTabStack.lookupLocal(routineName);
// Not already defined locally: Enter into the local symbol table.
if (routineId == null) {
routineId = symTabStack.enterLocal(routineName);
routineId.appendLineNumber(token.getLineNumber());
}
// If already defined, it should be a forward definition.
else {
routineId = null;
errorHandler.flag(token, IDENTIFIER_REDEFINED, this);
}
token = nextToken(); // consume routine name identifier
} else {
errorHandler.flag(token, MISSING_IDENTIFIER, this);
}
// If necessary, create a dummy routine name symbol table entry.
if (routineId == null) {
routineId = symTabStack.enterLocal(dummyName);
}
return routineId;
}
public static void main(String args[]) {
if (args.length != 1) {
System.err.println("Usage: java OCamlLexerTester <input.ml>");
System.exit(1);
}
OCamlLexer lexer = new OCamlLexer();
ArrayList<Token> tokens = new ArrayList<Token>();
try {
FileReader reader = new FileReader(args[0]);
lexer.tokenize(reader, tokens);
for (Token t : tokens)
System.out.println(
t.getType() + " - " + t.getStart() + "..." + (t.getStart() + t.getLength()));
} catch (IOException e) {
System.err.println("IO Exception");
e.printStackTrace();
}
}
/**
* Convert an arithmetic expression in infix notation (e.g 1+2) to postfix notation (e.g 12+) Only
* simple expressions not containing brackets are supported. <br>
* A description of the algorithm used to accomplish this task can be found at:
* http://scriptasylum.com/tutorials/infix_postfix/algorithms/infix-postfix/index.htm <br>
* Basically the list of {@code Tokens} is being scanned from left to right, all operands
* (numbers) are pushed onto a {@code Stack}. If the {@code Token} is an operator it is compared
* to the operator on top of the {@code Stack}. If the operator on top of the {@code Stack} has a
* higher or equal weight than the current operator, the top of the {@code Stack} is pushed to the
* postfix list. Therefore the operators with higher weight are going to be executed first when
* the expression is evaluated. This process continues until the top of the {@code Stack} has a
* lower weight than the current operator or the {@code Stack} is empty. Then the current operator
* is pushed onto the {@code Stack}.
*
* @param tokenizer The {@code Tokenizer} which will be converted to postfix. It must not be
* {@code null}.
* @return The resulting {@code List} of {@code Token}s in postfix order.
*/
private static List<Token> postfix(Tokenizer tokenizer) {
Stack<Token> stack = new Stack<>();
List<Token> postfix = new LinkedList<>();
BiPredicate<Token, Token> hasHigherOrEqualWeight =
(t1, t2) ->
Operators.getOpWeight(t1.getOperator()) - Operators.getOpWeight(t2.getOperator()) >= 0;
while (tokenizer.hasNext()) {
Token t = tokenizer.next();
switch (t.getType()) {
case NUMBER:
postfix.add(t);
break;
case OPERATOR:
if (!stack.isEmpty() && hasHigherOrEqualWeight.test(stack.peek(), t)) {
while (!stack.isEmpty() && hasHigherOrEqualWeight.test(stack.peek(), t)) {
postfix.add(stack.pop());
}
stack.push(t);
} else {
stack.push(t);
}
break;
case OPENING_BRACKET:
case CLOSING_BRACKET:
throw new IllegalArgumentException(
"Cannot create postfix expression if the source contains brackets.");
}
}
while (!stack.isEmpty()) postfix.add(stack.pop());
return postfix;
}
/**
* Consume and return the {@linkplain #getCurrentToken current symbol}.
*
* <p>E.g., given the following input with {@code A} being the current lookahead symbol, this
* function moves the cursor to {@code B} and returns {@code A}.
*
* <pre>
* A B
* ^
* </pre>
*
* If the parser is not in error recovery mode, the consumed symbol is added to the parse tree
* using {@link ParserRuleContext#addChild(Token)}, and {@link ParseTreeListener#visitTerminal} is
* called on any parse listeners. If the parser <em>is</em> in error recovery mode, the consumed
* symbol is added to the parse tree using {@link ParserRuleContext#addErrorNode(Token)}, and
* {@link ParseTreeListener#visitErrorNode} is called on any parse listeners.
*/
public Token consume() {
Token o = getCurrentToken();
if (o.getType() != EOF) {
getInputStream().consume();
}
boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty();
if (_buildParseTrees || hasListener) {
if (_errHandler.inErrorRecoveryMode(this)) {
ErrorNode node = _ctx.addErrorNode(o);
if (_parseListeners != null) {
for (ParseTreeListener listener : _parseListeners) {
listener.visitErrorNode(node);
}
}
} else {
TerminalNode node = _ctx.addChild(o);
if (_parseListeners != null) {
for (ParseTreeListener listener : _parseListeners) {
listener.visitTerminal(node);
}
}
}
}
return o;
}
public List<TerminalNode> getTokens(int ttype) {
if (children == null) {
return Collections.emptyList();
}
List<TerminalNode> tokens = null;
for (ParseTree o : children) {
if (o instanceof TerminalNode) {
TerminalNode tnode = (TerminalNode) o;
Token symbol = tnode.getSymbol();
if (symbol.getType() == ttype) {
if (tokens == null) {
tokens = new ArrayList<TerminalNode>();
}
tokens.add(tnode);
}
}
}
if (tokens == null) {
return Collections.emptyList();
}
return tokens;
}
/**
* Evaluate an expression in postfix notation. The expression must not contain brackes.
*
* <p>The algorithm used to perform the evaluation can be found at: algorithm from
* http://scriptasylum.com/tutorials/infix_postfix/algorithms/postfix-evaluation/index.htm
*
* <p>Basically the expression is iterated over from left to right, all number tokens ({@code
* double}) are pushed onto a {@code Stack} and all operator tokens are evaluated using the
* elements on the {@code Stack}. Note that the operators '+' and '-' are special as they can be
* unary (as opposed to binary) operators, which means it is possible for them to only operate on
* one operand.
*
* @param postfix The postfix expression to evaluate. It must not be {@code null} and it must not
* contain brackets.
* @return The result of the evaluation as a {@code double}
*/
private static double evaluatePostfix(List<Token> postfix) {
Stack<Double> stack = new Stack<>();
for (Token t : postfix) {
switch (t.getType()) {
case NUMBER:
stack.push(t.getNumber());
break;
case OPERATOR:
Double t1, t2;
t2 = stack.pop();
Double result = 0.0;
switch (t.getOperator()) {
case "+":
if (stack.isEmpty()) result = +t2;
else {
t1 = stack.pop();
result = t1 + t2;
}
break;
case "-":
if (stack.isEmpty()) result = -t2;
else {
t1 = stack.pop();
result = t1 - t2;
}
break;
case "*":
t1 = stack.pop();
result = t1 * t2;
break;
case "/":
t1 = stack.pop();
result = t1 / t2;
break;
case "%":
t1 = stack.pop();
result = t1 % t2;
break;
case "^":
t1 = stack.pop();
result = Math.pow(t1, t2);
break;
case "~":
result = (double) ~(long) (double) t2;
break;
default:
t1 = stack.pop();
result = Operators.getOperatorFunction((OperatorToken) t).apply(t1, t2);
}
stack.push(result);
break;
case OPENING_BRACKET:
case CLOSING_BRACKET:
throw new IllegalArgumentException(
"Cannot evaluate postfix expression if it contains brackets.");
}
}
return stack.pop();
}
/** logicalAnd ::= equality AND equality */
private Value logicalAnd() {
Value res = equality();
while (token.getType() == TokenType.AND) {
Value operand = equality();
res.setBooleanValue(res.getBooleanValue() && operand.getBooleanValue());
}
return res;
}
/** logicalOr ::= logicalAnd OR logicalAnd */
private Value logicalOr() {
Value res = logicalAnd();
while (token.getType() == TokenType.OR) {
Value operand = logicalAnd();
res.setBooleanValue(res.getBooleanValue() || operand.getBooleanValue());
}
return res;
}
private void testSingleValue(String statement, String expectedData, TokenType expectedTokenType) {
ArrayList<Token> tokens = Lexer.lex(statement);
Assert.assertEquals(1, tokens.size());
Token actualToken = tokens.get(0);
Assert.assertEquals(expectedData, actualToken.getData());
Assert.assertEquals(expectedTokenType, actualToken.getType());
}
/**
* Process a token. Handles common token processing functionality then delegates to the
* individual concrete handlers.
*
* @param ctx the current parse context
* @throws InvalidQueryException if unable to process the token
*/
public void handleToken(ParseContext ctx) throws InvalidQueryException {
Token token = ctx.getTokens()[ctx.getCurrentTokensIndex()];
if (!validate(ctx.getPreviousTokenType())) {
throw new InvalidQueryException(
"Unexpected token encountered in query string. Last Token Type="
+ ctx.getPreviousTokenType()
+ ", Current Token[type="
+ token.getType()
+ ", value='"
+ token.getValue()
+ "']");
}
ctx.setTokenType(token.getType());
int idxIncrement = _handleToken(ctx);
ctx.setCurrentTokensIndex(ctx.getCurrentTokensIndex() + idxIncrement);
}
public static void main(String[] args) throws Exception {
ANTLRFileStream input = new ANTLRFileStream(args[0]);
WigLexer lexer = new WigLexer(input);
Token token;
while ((token = lexer.nextToken()) != Token.EOF_TOKEN) {
System.out.println("Token: " + token.getText() + "(" + token.getType() + ")");
}
}
/**
* Get an integer option. Given the name of the option find its associated integer value. If the
* associated value is not an integer or is not in the table, then throw an exception of type
* NumberFormatException.
*
* @param key The name of the option
* @return The value associated with the key.
*/
public int getIntegerOption(String key) throws NumberFormatException {
Token t = (Token) options.get(key);
if (t == null || t.getType() != ANTLRTokenTypes.INT) {
throw new NumberFormatException();
} else {
return Integer.parseInt(t.getText());
}
}
public final TokenList cloneTokens() {
TokenList result = new TokenList();
for (Token t = fFirst; t != null; t = (Token) t.getNext()) {
if (t.getType() != CPreprocessor.tSCOPE_MARKER) {
result.append(t.clone());
}
}
return result;
}
public void lex() {
try {
tok = split.getToken();
text = tok.getString();
val = tok.getType();
} catch (Exception e) {
e.printStackTrace();
}
}
/** add n elements to buffer */
protected void fetch(int n) {
for (int i = 1; i <= n; i++) {
Token t = tokenSource.nextToken();
t.setTokenIndex(tokens.size());
// System.out.println("adding "+t+" at index "+tokens.size());
tokens.add(t);
if (t.getType() == Token.EOF) break;
}
}