public void testCreateTokensFromRealCode() throws Exception {
BufferedInputStream stream =
new BufferedInputStream(
new ByteArrayInputStream(
("TestCase.prototype.setUp = function(){"
+ " this.foo = 1;"
+ " /*:DOC bar = <div></div>*/"
+ "};")
.getBytes()));
BufferedTokenStream lexer = new HtmlDocLexer().createStream(stream);
List<Token> expected =
Lists.newArrayList(
ConcreteToken.from("TestCase.prototype.setUp "),
ConcreteToken.from("="),
ConcreteToken.from(" function()"),
ConcreteToken.from("{"),
ConcreteToken.from(" this.foo "),
tokens[3],
ConcreteToken.from(" 1; "),
tokens[0],
ConcreteToken.from(" bar "),
ConcreteToken.from("="),
ConcreteToken.from(" <div></div>"),
tokens[1],
tokens[4]);
for (Token token : expected) {
assertEquals(token, lexer.read());
}
}
public void testCreateTokensWithOtherTokens() throws Exception {
Token otherOne = new ConcreteToken("<div></div>".toCharArray());
BufferedInputStream stream =
new BufferedInputStream(
new ByteArrayInputStream(
(tokens[0].toString() + otherOne.toString() + tokens[1].toString()).getBytes()));
BufferedTokenStream lexer = new HtmlDocLexer().createStream(stream);
List<Token> expected = Lists.newArrayList(tokens[0], otherOne, tokens[1]);
for (Token token : expected) {
assertEquals(token, lexer.read());
}
}
public void create(BufferedTokenStream stream, Nodes nodes) {
if (!BLOCK_START_TOKEN.equals(stream.read())) {
stream.reset();
return;
}
nodes.add(new TextNode(BLOCK_START_TOKEN));
stream.mark();
while (stream.available()) {
Token token = stream.read();
if (BLOCK_END_TOKEN.equals(token)) {
stream.mark();
nodes.add(new TextNode(BLOCK_END_TOKEN));
return;
}
if (BLOCK_START_TOKEN.equals(token)) {
stream.reset();
create(stream, nodes);
return;
}
stream.reset();
for (NodeFactory factory : factories) {
factory.create(stream, nodes);
}
}
}
public void testCreateTokens() throws Exception {
List<Token> expected = Lists.newArrayList(tokens[0], tokens[1], tokens[2], tokens[3]);
BufferedInputStream stream =
new BufferedInputStream(
new ByteArrayInputStream(
(tokens[0].toString()
+ tokens[1].toString()
+ tokens[2].toString()
+ tokens[3].toString())
.getBytes()));
BufferedTokenStream lexer = new HtmlDocLexer().createStream(stream);
for (Token token : expected) {
assertEquals(token, lexer.read());
}
assertFalse(lexer.available());
}