public static void main(String[] args) {
try {
InputStream is = StreamTokenering.class.getResourceAsStream("/input.txt");
StreamTokenizer in = new StreamTokenizer(new InputStreamReader(is));
in.ordinaryChar('.');
in.ordinaryChar('\'');
int wordCount = 0, numCount = 0, punctionCount = 0, count = 0;
double token;
while ((token = in.nextToken()) != StreamTokenizer.TT_EOF) {
count++;
if (token == StreamTokenizer.TT_WORD) {
wordCount++;
} else if (token == StreamTokenizer.TT_NUMBER) {
numCount++;
} else {
punctionCount++;
}
System.out.println(in.toString());
}
System.out.println("单词总数为：" + count);
System.out.println("单词数为：" + wordCount);
System.out.println("数字数为：" + numCount);
System.out.println("标点符号数为：" + punctionCount++);
} catch (IOException e) {
e.printStackTrace();
}
}
public CSVReader(BufferedReader input, char customizedSeparator) {
this.separator = customizedSeparator;
parser = new StreamTokenizer(input);
parser.ordinaryChars(0, 255);
parser.wordChars(0, 255);
parser.ordinaryChar('\"');
parser.ordinaryChar(customizedSeparator);
// Need to do set EOL significance after setting ordinary and word
// chars, and need to explicitly set \n and \r as whitespace chars
// for EOL detection to work
parser.eolIsSignificant(true);
parser.whitespaceChars('\n', '\n');
parser.whitespaceChars('\r', '\r');
atEOF = false;
}
/** Create an HTTP tokenizer, given a StreamTokenizer for the web page. */
public HttpTokenizer(StreamTokenizer tokens) throws IOException {
// Create a stream tokenizer
this.tokens = tokens;
// Set up the appropriate defaults
tokens.eolIsSignificant(false);
tokens.lowerCaseMode(true);
tokens.wordChars('<', '<');
tokens.wordChars('>', '>');
tokens.wordChars('/', '/');
tokens.wordChars('=', '=');
tokens.wordChars('@', '@');
tokens.wordChars('!', '!');
tokens.wordChars('-', '-');
tokens.ordinaryChar('.');
tokens.ordinaryChar('?');
}
/**
* Return an interned VarInfoAux that represents a given string. Elements are separated by commas,
* in the form:
*
* <p>x = a, "a key" = "a value"
*
* <p>Parse allow for quoted elements. White space to the left and right of keys and values do not
* matter, but inbetween does.
*/
public static /*@Interned*/ VarInfoAux parse(String inString) throws IOException {
Reader inStringReader = new StringReader(inString);
StreamTokenizer tok = new StreamTokenizer(inStringReader);
tok.resetSyntax();
tok.wordChars(0, Integer.MAX_VALUE);
tok.quoteChar('\"');
tok.whitespaceChars(' ', ' ');
tok.ordinaryChar('[');
tok.ordinaryChar(']');
tok.ordinaryChars(',', ',');
tok.ordinaryChars('=', '=');
Map</*@Interned*/ String, /*@Interned*/ String> map = theDefault.map;
String key = "";
String value = "";
boolean seenEqual = false;
boolean insideVector = false;
for (int tokInfo = tok.nextToken();
tokInfo != StreamTokenizer.TT_EOF;
tokInfo = tok.nextToken()) {
@SuppressWarnings("interning") // initialization-checking pattern
boolean mapUnchanged = (map == theDefault.map);
if (mapUnchanged) {
// We use default values if none are specified. We initialize
// here rather than above to save time when there are no tokens.
map = new HashMap</*@Interned*/ String, /*@Interned*/ String>(theDefault.map);
}
/*@Interned*/ String token;
if (tok.ttype == StreamTokenizer.TT_WORD || tok.ttype == '\"') {
assert tok.sval != null
: "@AssumeAssertion(nullness): representation invariant of StreamTokenizer";
token = tok.sval.trim().intern();
} else {
token = ((char) tok.ttype + "").intern();
}
debug.fine("Token info: " + tokInfo + " " + token);
if (token == "[") { // interned
if (!seenEqual) throw new IOException("Aux option did not contain an '='");
if (insideVector) throw new IOException("Vectors cannot be nested in an aux option");
if (value.length() > 0) throw new IOException("Cannot mix scalar and vector values");
insideVector = true;
value = "";
} else if (token == "]") { // interned
if (!insideVector) throw new IOException("']' without preceding '['");
insideVector = false;
} else if (token == ",") { // interned
if (!seenEqual) throw new IOException("Aux option did not contain an '='");
if (insideVector) throw new IOException("',' cannot be used inside a vector");
map.put(key.intern(), value.intern());
key = "";
value = "";
seenEqual = false;
} else if (token == "=") { // interned
if (seenEqual) throw new IOException("Aux option contained more than one '='");
if (insideVector) throw new IOException("'=' cannot be used inside a vector");
seenEqual = true;
} else {
if (!seenEqual) {
key = (key + " " + token).trim();
} else if (insideVector) {
value = value + " \"" + token.trim() + "\"";
} else {
value = (value + " " + token).trim();
}
}
}
if (seenEqual) {
map.put(key.intern(), value.intern());
}
// Interning
VarInfoAux result = new VarInfoAux(map).intern();
assert interningMap != null
: "@AssumeAssertion(nullness): application invariant: postcondition of intern(), which was just called";
if (debug.isLoggable(Level.FINE)) {
debug.fine("New parse " + result);
debug.fine("Intern table size: " + new Integer(interningMap.size()));
}
return result;
}
