@Test
public void shouldSupportNestedOpenCloseBlocks() throws Exception {
final Lexer lexer =
new Lexer(
"{% block1 %}{% block2 %}{% inner block2 %}{% endblock2 %}{% inner block1 %}{% endblock1 %}");
final Parser parser = new Parser(lexer.tokens());
final List<Node> nodes = parser.parse();
assertNotNull(nodes);
assertEquals(1, nodes.size());
assertTrue(nodes.get(0) instanceof BlockNode);
BlockNode block1 = (BlockNode) nodes.get(0);
assertEquals("block1", block1.getName());
assertEquals(2, block1.getNodes().size());
assertTrue(block1.getNodes().get(0) instanceof BlockNode);
assertTrue(block1.getNodes().get(1) instanceof BlockNode);
BlockNode block2 = (BlockNode) block1.getNodes().get(0);
BlockNode innerBlock1 = (BlockNode) block1.getNodes().get(1);
assertEquals("block2", block2.getName());
assertEquals("inner", innerBlock1.getName());
assertEquals(1, block2.getNodes().size());
assertTrue(block2.getNodes().get(0) instanceof BlockNode);
BlockNode innerBlock2 = (BlockNode) block2.getNodes().get(0);
assertEquals("inner", innerBlock2.getName());
}
public Rexpr(int[] loopStart) throws ParserException {
expr1 = new Expr();
switch (Lexer.nextToken) {
case Token.EQ_OP:
Op = "if_icmpne";
Lexer.lex();
break;
case Token.NOT_EQ:
Op = "if_icmpeq";
Lexer.lex();
break;
case Token.LESSER_OP:
Op = "if_icmpge";
Lexer.lex();
break;
case Token.GREATER_OP:
Op = "if_icmple";
Lexer.lex();
break;
default:
throw new ParserException(Lexer.nextChar);
}
expr2 = new Expr();
loopStart[0] = Code.codeptr;
Code.gen(Op, 3);
}
public static void main(String[] args) {
String inFile = "C:\\Users\\Sukoon\\Documents\\NetBeansProjects\\cdproj\\src\\Sample.in";
String outFile = "C:\\Users\\Sukoon\\Desktop\\Sample.out";
if (args.length > 1) {
inFile = args[0];
outFile = args[1];
}
Lexer lexer = new Lexer(inFile);
try {
BufferedWriter writer = new BufferedWriter(new FileWriter(outFile));
Token t;
while ((t = lexer.nextToken()) != null) {
writer.write(t.toString());
writer.newLine();
}
writer.close();
System.out.println("Done tokenizing file: " + inFile);
System.out.println("Output written in file: " + outFile);
} catch (IOException e) {
e.printStackTrace();
}
}
public Assign() throws ParserException {
int idnum;
if (Code.variables.containsKey(Lexer.ident)) {
idnum = Code.variables.get(Lexer.ident);
} else throw new ParserException(Lexer.nextChar);
Lexer.lex();
if (Lexer.nextToken == Token.ASSIGN_OP) {
Lexer.lex();
} else throw new ParserException(Lexer.nextChar);
expr = new Expr();
if (Lexer.nextToken == Token.SEMICOLON) {
Lexer.lex();
} else if (Lexer.nextToken == Token.RIGHT_PAREN) {
Lexer.lex();
} else throw new ParserException(Lexer.nextChar);
if (!Loop.inc) {
if (idnum < 3) {
Code.gen("istore_" + (idnum + 1));
} else {
Code.gen("istore_" + (idnum + 1), 2);
}
} else {
Loop.storeValue[Loop.incpt] = "istore_" + (idnum + 1);
Loop.incpt++;
Loop.inc = false;
}
}
protected String hvalue() throws ParseException {
StringBuffer retval = new StringBuffer();
while (lexer.hasMoreChars()) {
char la = lexer.lookAhead(0);
// Look for a character that can terminate a URL.
if (la == '+'
|| la == '?'
|| la == ':'
|| la == '['
|| la == ']'
|| la == '/'
|| la == '$'
|| la == '_'
|| la == '-'
|| la == '"'
|| la == '!'
|| la == '~'
|| la == '*'
|| la == '.'
|| la == '('
|| la == ')'
|| Lexer.isAlpha(la)
|| Lexer.isDigit(la)) {
lexer.consume(1);
retval.append(la);
} else if (la == '%') {
retval.append(escaped());
} else break;
}
return retval.toString();
}
private Token tokenpaste(Token arg1, Token arg2, PreprocessorMacro macro) {
if (arg1 == null) {
return arg2;
}
if (arg2 == null) {
return arg1;
}
final char[] image1 = arg1.getCharImage();
final char[] image2 = arg2.getCharImage();
final int l1 = image1.length;
final int l2 = image2.length;
final char[] image = new char[l1 + l2];
System.arraycopy(image1, 0, image, 0, l1);
System.arraycopy(image2, 0, image, l1, l2);
Lexer lex = new Lexer(image, fLexOptions, ILexerLog.NULL, null);
try {
Token t1 = lex.nextToken();
Token t2 = lex.nextToken();
if (t1.getType() != IToken.tEND_OF_INPUT && t2.getType() == IToken.tEND_OF_INPUT) {
t1.setOffset(arg1.getOffset(), arg2.getEndOffset());
return t1;
}
} catch (OffsetLimitReachedException e) {
}
handleProblem(IProblem.PREPROCESSOR_MACRO_PASTING_ERROR, macro.getNameCharArray());
return null;
}
// statement <- COLUMN COMPARE VALUE
// | COLUMN IS NULL
private static void parseStatement(Lexer lexer) {
// both possibilities start with COLUMN
if (lexer.currentToken() != Lexer.TOKEN_COLUMN) {
throw new IllegalArgumentException("syntax error, expected column name");
}
lexer.advance();
// statement <- COLUMN COMPARE VALUE
if (lexer.currentToken() == Lexer.TOKEN_COMPARE) {
lexer.advance();
if (lexer.currentToken() != Lexer.TOKEN_VALUE) {
throw new IllegalArgumentException("syntax error, expected quoted string");
}
lexer.advance();
return;
}
// statement <- COLUMN IS NULL
if (lexer.currentToken() == Lexer.TOKEN_IS) {
lexer.advance();
if (lexer.currentToken() != Lexer.TOKEN_NULL) {
throw new IllegalArgumentException("syntax error, expected NULL");
}
lexer.advance();
return;
}
// didn't get anything good after COLUMN
throw new IllegalArgumentException("syntax error after column name");
}
public static void main(String[] args) throws InterruptedException {
ArrayList<File> fileList = Utils.listFiles(IN_DIR);
for (File file : fileList) {
System.out.println("Scanning file " + file.getPath());
LexIO io = new LexIO(file);
LexIO noTrash = Lexer.removeTrash(io);
LexIO noComments = Lexer.removeComments(noTrash);
LexIO noMalformed = Lexer.removeMalformed(noComments);
LexIO tokens = Lexer.getValidTokens(noMalformed);
// tokens.writeOutput(OUT_DIR + "/" + file.getName());
// for(Token token: tokens.getTokens()) {
// System.out.println( token.getRepresentation() + " " + token.getType() + " " +
// token.getLine());
// }
//
// Thread.sleep(100); // used for debug
// for(TokenError error : tokens.getErrors()) {
// System.err.println( error.getRepresentation() + " " + error.getType() + " " + " " +
// error.getLine());
// }
SemanticAnalyzer analyzer = new SemanticAnalyzer(tokens);
analyzer.parseProgram();
analyzer.writeErrorOutput(OUT_DIR + "/" + file.getName());
}
}
private void variableDeclaration(SymbolTable sym) throws ParseException {
start("variableDeclaration");
if (!lex.match("var")) throw new ParseException(15);
lex.nextLex();
nameDeclaration(sym);
stop("variableDeclaration");
}
private Lexeme getLexeme(Lexer lex) throws ValidationReaderException {
Lexeme lexeme = null;
if (lex.hasLexeme()) {
lexeme = lex.nextLexeme();
}
return lexeme;
}
@Test
public void should_print_at_start() {
Lexer lexer = new Lexer("(3x + y) * 3");
// (
assertEquals("[(]3x + y) * 3", lexer.printPosition());
}
public static void main(String argv[]) throws java.io.IOException {
for (int i = 0; i < argv.length; ++i) {
String filename = argv[i];
if (argv.length > 1) System.out.println("***Processing: " + filename);
ErrorMsg.ErrorMsg errorMsg = new ErrorMsg.ErrorMsg(filename);
java.io.InputStream inp = new java.io.FileInputStream(filename);
Lexer lexer = new Yylex(inp, errorMsg);
java_cup.runtime.Symbol tok;
do {
String extra = "";
tok = lexer.nextToken();
switch (tok.sym) {
case sym.ID:
extra = "\t$" + tok.value;
break;
case sym.INT:
extra = "\t#" + tok.value;
break;
case sym.STRING:
extra = " \"" + tok.value + "\"";
break;
}
System.out.println(symnames[tok.sym] + " " + tok.left + extra);
} while (tok.sym != sym.EOF);
inp.close();
}
}
private List<Lexeme> getLexemes(Lexer lex) throws ValidationReaderException {
List<Lexeme> lexemes = new ArrayList<Lexeme>();
while (lex.hasLexeme()) {
lexemes.add(lex.nextLexeme());
}
return lexemes;
}
@Test
public void should_print_at_last_letter() throws ValidationReaderException {
Lexer lexer = new Lexer("(3x + y) * 3");
while (lexer.hasLexeme()) {
lexer.nextLexeme();
}
assertEquals("(3x + y) * 3[]", lexer.printPosition());
}
@Test
public void should_print_at_2nd_letter() throws ValidationReaderException {
Lexer lexer = new Lexer("(3x + y) * 3");
// (
lexer.nextLexeme(); // 3
assertEquals("([3]x + y) * 3", lexer.printPosition());
}
@Test
public void should_return_EndOfDefinition() throws ValidationReaderException {
Lexer lex = new Lexer("x1 23");
lex.nextLexeme();
lex.nextLexeme();
assertThat(EOD, "", lex.nextLexeme());
}
protected void acceptIdentifier(String text) {
if (identifierEquals(text)) {
lexer.nextToken();
} else {
setErrorEndPos(lexer.pos());
throw new ParserException("syntax error, expect " + text + ", actual " + lexer.token());
}
}
private void nonFunctionDeclaration(SymbolTable sym) throws ParseException {
start("nonFunctionDeclaration");
if (lex.match("var")) variableDeclaration(sym);
else if (lex.match("const")) constantDeclaration(sym);
else if (lex.match("type")) typeDeclaration(sym);
else throw new ParseException(26); // TODO: probbly wrong exception
stop("nonFunctionDeclaration");
}
public ErrorCorrectingParser(Scanner lexer, int whitespaceMask) {
if (lexer instanceof Lexer) {
Lexer eglLexer = (Lexer) lexer;
eglLexer.returnBlockComments = (whitespaceMask & RETURN_BLOCK_COMMENT) != 0;
eglLexer.returnLineBreaks = (whitespaceMask & RETURN_LINEBREAKS) != 0;
eglLexer.returnLineComments = (whitespaceMask & RETURN_LINE_COMMENT) != 0;
}
stream = new TokenStream(RECOVERY_SUCCESS + AdvancedPhraseRecovery.INPUT_DELETION_LIMIT, lexer);
}
private void tokenExpected(TokenType type, String text) {
myLexer.nextToken();
if (type != myLexer.getTokenType()) {
throw new ParsingException(type.getName() + " expected but " + myLexer.getToken() + " found");
}
if (text != null && !text.equals(myLexer.getToken())) {
throw new ParsingException(text + " expected but " + myLexer.getToken() + " found");
}
}
private void reference(SymbolTable sym) throws ParseException {
start("reference");
if (!lex.isIdentifier()) {
throw new ParseException(27); // expected an identifier
}
lex.nextLex();
referencePrime(sym);
stop("reference");
}
private void timesExpression(SymbolTable sym) throws ParseException {
start("timesExpression");
term(sym);
while (lex.match("*") || lex.match("/") || lex.match("%")) {
lex.nextLex();
term(sym);
}
stop("timesExpression");
}
private void program(SymbolTable sym) throws ParseException {
start("program");
while (lex.tokenCategory() != Lexer.endOfInput) {
declaration(sym);
if (lex.match(";")) lex.nextLex();
else throw new ParseException(18);
}
stop("program");
}
private void expression(SymbolTable sym) throws ParseException {
start("expression");
relExpression(sym);
while (lex.match("and") || lex.match("or")) {
lex.nextLex();
relExpression(sym);
}
stop("expression");
}
private void plusExpression(SymbolTable sym) throws ParseException {
start("plusExpression");
timesExpression(sym);
while (lex.match("+") || lex.match("-") || lex.match("<<")) {
lex.nextLex();
timesExpression(sym);
}
stop("plusExpression");
}
private Type returnType(SymbolTable sym) throws ParseException {
start("returnType");
if (lex.match(":")) {
lex.nextLex();
return type(sym);
}
stop("returnType");
return PrimitiveType.VoidType;
}
// TODO not clear what arguments should do and what argumentList should do :(
private void arguments(SymbolTable sym) throws ParseException {
start("arguments");
if (!lex.match("(")) throw new ParseException(21);
lex.nextLex();
argumentList(sym);
if (!lex.match(")")) throw new ParseException(22);
lex.nextLex();
stop("arguments");
}
private void classDeclaration(SymbolTable sym) throws ParseException {
start("classDeclaration");
if (!lex.match("class")) throw new ParseException(5);
lex.nextLex();
if (!lex.isIdentifier()) throw new ParseException(27);
lex.nextLex();
classBody(sym);
stop("classDeclaration");
}
private List<Expression> parseExpressionList() {
List<Expression> expressions = new ArrayList<Expression>();
do {
Expression expression = parseExpression();
expressions.add(expression);
} while (myLexer.nextToken() == TokenType.COMMA);
myLexer.pushBack();
return expressions;
}
public Parser(Lexer lexer, String[] keywords) {
this.lexer = lexer;
for (String keyword : keywords) {
this.keywords.add(keyword);
}
token = lexer.nextToken();
if (token.type == TokenType.EOF) lookAhead = token;
else lookAhead = lexer.nextToken();
}
