public void create(BufferedTokenStream stream, Nodes nodes) { if (!BLOCK_START_TOKEN.equals(stream.read())) { stream.reset(); return; } nodes.add(new TextNode(BLOCK_START_TOKEN)); stream.mark(); while (stream.available()) { Token token = stream.read(); if (BLOCK_END_TOKEN.equals(token)) { stream.mark(); nodes.add(new TextNode(BLOCK_END_TOKEN)); return; } if (BLOCK_START_TOKEN.equals(token)) { stream.reset(); create(stream, nodes); return; } stream.reset(); for (NodeFactory factory : factories) { factory.create(stream, nodes); } } }
public void testCreateTokens() throws Exception { List<Token> expected = Lists.newArrayList(tokens[0], tokens[1], tokens[2], tokens[3]); BufferedInputStream stream = new BufferedInputStream( new ByteArrayInputStream( (tokens[0].toString() + tokens[1].toString() + tokens[2].toString() + tokens[3].toString()) .getBytes())); BufferedTokenStream lexer = new HtmlDocLexer().createStream(stream); for (Token token : expected) { assertEquals(token, lexer.read()); } assertFalse(lexer.available()); }