public void testCreateTokensFromRealCode() throws Exception {
    BufferedInputStream stream =
        new BufferedInputStream(
            new ByteArrayInputStream(
                ("TestCase.prototype.setUp = function(){"
                        + "  this.foo = 1;"
                        + "  /*:DOC bar = <div></div>*/"
                        + "};")
                    .getBytes()));

    BufferedTokenStream lexer = new HtmlDocLexer().createStream(stream);
    List<Token> expected =
        Lists.newArrayList(
            ConcreteToken.from("TestCase.prototype.setUp "),
            ConcreteToken.from("="),
            ConcreteToken.from(" function()"),
            ConcreteToken.from("{"),
            ConcreteToken.from("  this.foo "),
            tokens[3],
            ConcreteToken.from(" 1;  "),
            tokens[0],
            ConcreteToken.from(" bar "),
            ConcreteToken.from("="),
            ConcreteToken.from(" <div></div>"),
            tokens[1],
            tokens[4]);
    for (Token token : expected) {
      assertEquals(token, lexer.read());
    }
  }
  public void testCreateTokensWithOtherTokens() throws Exception {
    Token otherOne = new ConcreteToken("<div></div>".toCharArray());
    BufferedInputStream stream =
        new BufferedInputStream(
            new ByteArrayInputStream(
                (tokens[0].toString() + otherOne.toString() + tokens[1].toString()).getBytes()));

    BufferedTokenStream lexer = new HtmlDocLexer().createStream(stream);
    List<Token> expected = Lists.newArrayList(tokens[0], otherOne, tokens[1]);
    for (Token token : expected) {
      assertEquals(token, lexer.read());
    }
  }
 public void create(BufferedTokenStream stream, Nodes nodes) {
   if (!BLOCK_START_TOKEN.equals(stream.read())) {
     stream.reset();
     return;
   }
   nodes.add(new TextNode(BLOCK_START_TOKEN));
   stream.mark();
   while (stream.available()) {
     Token token = stream.read();
     if (BLOCK_END_TOKEN.equals(token)) {
       stream.mark();
       nodes.add(new TextNode(BLOCK_END_TOKEN));
       return;
     }
     if (BLOCK_START_TOKEN.equals(token)) {
       stream.reset();
       create(stream, nodes);
       return;
     }
     stream.reset();
     for (NodeFactory factory : factories) {
       factory.create(stream, nodes);
     }
   }
 }
 public void testCreateTokens() throws Exception {
   List<Token> expected = Lists.newArrayList(tokens[0], tokens[1], tokens[2], tokens[3]);
   BufferedInputStream stream =
       new BufferedInputStream(
           new ByteArrayInputStream(
               (tokens[0].toString()
                       + tokens[1].toString()
                       + tokens[2].toString()
                       + tokens[3].toString())
                   .getBytes()));
   BufferedTokenStream lexer = new HtmlDocLexer().createStream(stream);
   for (Token token : expected) {
     assertEquals(token, lexer.read());
   }
   assertFalse(lexer.available());
 }