public Token nextToken() throws InvalidCodeException {
str = str.trim();
if (pushBack) {
pushBack = false;
return lastToken;
}
if (str.equals("")) {
return lastToken = new Token(Token.TokenType.EMPTY);
}
for (TokenData data : tokenDatas) {
Matcher matcher = data.getPattern().matcher(str);
if (matcher.find()) {
String token = matcher.group().trim();
str = matcher.replaceFirst("");
if (data.getTokenType() == Token.TokenType.STRING_LITERAL) {
return lastToken =
new Token(Token.TokenType.STRING_LITERAL, token.substring(1, token.length() - 1));
} else {
return lastToken = new Token(data.getTokenType(), token);
}
}
}
throw new InvalidCodeException("Unknown token: " + str.substring(0, str.indexOf(" ")));
}
private List<IndexedEntry> getOrCreateEntries(
final String token, final EntryTypeName entryTypeName) {
final TokenData tokenData = getOrCreateTokenData(token);
List<IndexedEntry> entries = tokenData.typeToEntries.get(entryTypeName);
if (entryTypeName.mainWord) {
tokenData.hasMainEntry = true;
}
if (entries == null) {
entries = new ArrayList<IndexedEntry>();
tokenData.typeToEntries.put(entryTypeName, entries);
}
return entries;
}
