/** Create an HTTP tokenizer, given a StreamTokenizer for the web page. */ public HttpTokenizer(StreamTokenizer tokens) throws IOException { // Create a stream tokenizer this.tokens = tokens; // Set up the appropriate defaults tokens.eolIsSignificant(false); tokens.lowerCaseMode(true); tokens.wordChars('<', '<'); tokens.wordChars('>', '>'); tokens.wordChars('/', '/'); tokens.wordChars('=', '='); tokens.wordChars('@', '@'); tokens.wordChars('!', '!'); tokens.wordChars('-', '-'); tokens.ordinaryChar('.'); tokens.ordinaryChar('?'); }