private void parseTokens() {
if (!tokens.isEmpty()) {
return;
}
int startOffset = scanner.offset();
int startColumn = column;
buffer.setLength(0);
while (!scanner.eof() && isWhitespace(scanner.peek())) {
final char c = scanner.next();
column++;
if (!ignoreWhitespace) {
buffer.append(c);
}
}
if (buffer.length() > 0) {
addToken(TokenType.WHITESPACE, buffer.toString(), startOffset, line, startColumn);
return;
}
startOffset = scanner.offset();
outer:
while (!scanner.eof()) {
final char c = scanner.next();
if (isWhitespace(c)) {
scanner.pushBack();
break;
}
column++;
if (c == '-') {
parseBuffer(startOffset, startColumn);
addToken(TokenType.OPERATOR, c, scanner.offset() - 1, line, column);
break outer;
}
if (OperatorType.mayBeOperator(buffer.toString() + c)) {
buffer.append(c);
continue;
} else if (OperatorType.getExactMatch(buffer.toString()) != null) {
addToken(TokenType.OPERATOR, buffer.toString(), startOffset, line, startColumn);
buffer.setLength(0);
scanner.pushBack();
column--;
break;
} else if (OperatorType.mayBeOperator(c)) {
scanner.pushBack();
column--;
break;
}
switch (c) {
case '(':
parseBuffer(startOffset, startColumn);
addToken(TokenType.PARENS_OPEN, c, scanner.offset() - 1, line, column);
break outer;
case ')':
parseBuffer(startOffset, startColumn);
addToken(TokenType.PARENS_CLOSE, c, scanner.offset() - 1, line, column);
break outer;
case '=':
parseBuffer(startOffset, startColumn);
addToken(TokenType.EQUALS, c, scanner.offset() - 1, line, column);
break outer;
case ';':
parseBuffer(startOffset, startColumn);
addToken(TokenType.SEMICOLON, c, scanner.offset() - 1, line, column);
break outer;
case '\'':
parseBuffer(startOffset, startColumn);
addToken(TokenType.SINGLE_QUOTE, c, scanner.offset() - 1, line, column);
break outer;
case '"':
parseBuffer(startOffset, startColumn);
addToken(TokenType.DOUBLE_QUOTE, c, scanner.offset() - 1, line, column);
break outer;
case '.':
parseBuffer(startOffset, startColumn);
addToken(TokenType.DOT, c, scanner.offset() - 1, line, column);
break outer;
case '#':
parseBuffer(startOffset, startColumn);
addToken(TokenType.HASH, c, scanner.offset() - 1, line, column);
break outer;
case ',':
parseBuffer(startOffset, startColumn);
addToken(TokenType.COMMA, c, scanner.offset() - 1, line, column);
break outer;
case '\n':
parseBuffer(startOffset, startColumn);
addToken(TokenType.EOL, c, scanner.offset() - 1, line, column);
break outer;
case ':':
parseBuffer(startOffset, startColumn);
addToken(TokenType.COLON, ':', scanner.offset() - 1, line, column);
break outer;
case '\r':
parseBuffer(startOffset, startColumn);
if (!scanner.eof() && scanner.peek() == '\n') {
scanner.next();
addToken(TokenType.EOL, "\r\n", scanner.offset() - 2, line, column);
column++;
} else {
addToken(TokenType.EOL, "\r", scanner.offset() - 1, line, column);
}
line++;
column = 0;
break outer;
default:
buffer.append(c);
}
}
parseBuffer(startOffset, startColumn);
if (scanner.eof()) {
addToken(TokenType.EOF, "", scanner.offset(), line, column);
}
}
