/** Returns a concatenation of the remaining strings from a Tokenizer. */
private String remainingStrings() throws IOException {
StringBuffer buffer = null;
while (true) {
Tokenizer.Token t = get();
if (!t.isString()) break;
if (buffer == null) buffer = new StringBuffer();
buffer.append(t.value);
}
unget();
if (buffer == null) return null;
return buffer.toString();
}
private Token set(int type, StringBuffer value) {
if (type < 0) throw new IllegalArgumentException();
this.type = type;
this.value = value == null ? null : value.toString();
return this;
}
/**
* Gets the next token from a tokenizer.
*
* @param wantWhitespace If true, leading whitespace will be returned as a token.
* @param wantComment If true, comments are returned as tokens.
* @return The next token in the stream.
* @throws TextParseException The input was invalid.
* @throws IOException An I/O error occurred.
*/
public Token get(boolean wantWhitespace, boolean wantComment) throws IOException {
int type;
int c;
if (ungottenToken) {
ungottenToken = false;
if (current.type == WHITESPACE) {
if (wantWhitespace) return current;
} else if (current.type == COMMENT) {
if (wantComment) return current;
} else {
if (current.type == EOL) line++;
return current;
}
}
int skipped = skipWhitespace();
if (skipped > 0 && wantWhitespace) return current.set(WHITESPACE, null);
type = IDENTIFIER;
sb.setLength(0);
while (true) {
c = getChar();
if (c == -1 || delimiters.indexOf(c) != -1) {
if (c == -1) {
if (quoting) throw exception("EOF in " + "quoted string");
else if (sb.length() == 0) return current.set(EOF, null);
else return current.set(type, sb);
}
if (sb.length() == 0 && type != QUOTED_STRING) {
if (c == '(') {
multiline++;
skipWhitespace();
continue;
} else if (c == ')') {
if (multiline <= 0) throw exception("invalid " + "close " + "parenthesis");
multiline--;
skipWhitespace();
continue;
} else if (c == '"') {
if (!quoting) {
quoting = true;
delimiters = quotes;
type = QUOTED_STRING;
} else {
quoting = false;
delimiters = delim;
skipWhitespace();
}
continue;
} else if (c == '\n') {
return current.set(EOL, null);
} else if (c == ';') {
while (true) {
c = getChar();
if (c == '\n' || c == -1) break;
sb.append((char) c);
}
if (wantComment) {
ungetChar(c);
return current.set(COMMENT, sb);
} else if (c == -1 && type != QUOTED_STRING) {
checkUnbalancedParens();
return current.set(EOF, null);
} else if (multiline > 0) {
skipWhitespace();
sb.setLength(0);
continue;
} else return current.set(EOL, null);
} else throw new IllegalStateException();
} else ungetChar(c);
break;
} else if (c == '\\') {
c = getChar();
if (c == -1) throw exception("unterminated escape sequence");
sb.append('\\');
} else if (quoting && c == '\n') {
throw exception("newline in quoted string");
}
sb.append((char) c);
}
if (sb.length() == 0 && type != QUOTED_STRING) {
checkUnbalancedParens();
return current.set(EOF, null);
}
return current.set(type, sb);
}
