/** Create an HTTP tokenizer, given a StreamTokenizer for the web page. */
public HttpTokenizer(StreamTokenizer tokens) throws IOException {
// Create a stream tokenizer
this.tokens = tokens;
// Set up the appropriate defaults
tokens.eolIsSignificant(false);
tokens.lowerCaseMode(true);
tokens.wordChars('<', '<');
tokens.wordChars('>', '>');
tokens.wordChars('/', '/');
tokens.wordChars('=', '=');
tokens.wordChars('@', '@');
tokens.wordChars('!', '!');
tokens.wordChars('-', '-');
tokens.ordinaryChar('.');
tokens.ordinaryChar('?');
}
