private void parseBuffer(int startOffset, int startColumn) { if (buffer.length() == 0) { return; } final String value = buffer.toString(); buffer.setLength(0); if (OperatorType.getExactMatch(value) != null) { addToken(TokenType.OPERATOR, value, startOffset, line, startColumn); return; } if ("=".equals(value)) { addToken(TokenType.EQUALS, "=", scanner.offset() - 1, line, startColumn); return; } boolean isOnlyDigits = true; for (int i = 0, len = value.length(); i < len; i++) { if (!Character.isDigit(value.charAt(i))) { isOnlyDigits = false; break; } } if (isOnlyDigits) { addToken(TokenType.DIGITS, value, startOffset, line, startColumn); return; } addToken(TokenType.TEXT, value, startOffset, line, startColumn); }
private void parseTokens() { if (!tokens.isEmpty()) { return; } int startOffset = scanner.offset(); int startColumn = column; buffer.setLength(0); while (!scanner.eof() && isWhitespace(scanner.peek())) { final char c = scanner.next(); column++; if (!ignoreWhitespace) { buffer.append(c); } } if (buffer.length() > 0) { addToken(TokenType.WHITESPACE, buffer.toString(), startOffset, line, startColumn); return; } startOffset = scanner.offset(); outer: while (!scanner.eof()) { final char c = scanner.next(); if (isWhitespace(c)) { scanner.pushBack(); break; } column++; if (c == '-') { parseBuffer(startOffset, startColumn); addToken(TokenType.OPERATOR, c, scanner.offset() - 1, line, column); break outer; } if (OperatorType.mayBeOperator(buffer.toString() + c)) { buffer.append(c); continue; } else if (OperatorType.getExactMatch(buffer.toString()) != null) { addToken(TokenType.OPERATOR, buffer.toString(), startOffset, line, startColumn); buffer.setLength(0); scanner.pushBack(); column--; break; } else if (OperatorType.mayBeOperator(c)) { scanner.pushBack(); column--; break; } switch (c) { case '(': parseBuffer(startOffset, startColumn); addToken(TokenType.PARENS_OPEN, c, scanner.offset() - 1, line, column); break outer; case ')': parseBuffer(startOffset, startColumn); addToken(TokenType.PARENS_CLOSE, c, scanner.offset() - 1, line, column); break outer; case '=': parseBuffer(startOffset, startColumn); addToken(TokenType.EQUALS, c, scanner.offset() - 1, line, column); break outer; case ';': parseBuffer(startOffset, startColumn); addToken(TokenType.SEMICOLON, c, scanner.offset() - 1, line, column); break outer; case '\'': parseBuffer(startOffset, startColumn); addToken(TokenType.SINGLE_QUOTE, c, scanner.offset() - 1, line, column); break outer; case '"': parseBuffer(startOffset, startColumn); addToken(TokenType.DOUBLE_QUOTE, c, scanner.offset() - 1, line, column); break outer; case '.': parseBuffer(startOffset, startColumn); addToken(TokenType.DOT, c, scanner.offset() - 1, line, column); break outer; case '#': parseBuffer(startOffset, startColumn); addToken(TokenType.HASH, c, scanner.offset() - 1, line, column); break outer; case ',': parseBuffer(startOffset, startColumn); addToken(TokenType.COMMA, c, scanner.offset() - 1, line, column); break outer; case '\n': parseBuffer(startOffset, startColumn); addToken(TokenType.EOL, c, scanner.offset() - 1, line, column); break outer; case ':': parseBuffer(startOffset, startColumn); addToken(TokenType.COLON, ':', scanner.offset() - 1, line, column); break outer; case '\r': parseBuffer(startOffset, startColumn); if (!scanner.eof() && scanner.peek() == '\n') { scanner.next(); addToken(TokenType.EOL, "\r\n", scanner.offset() - 2, line, column); column++; } else { addToken(TokenType.EOL, "\r", scanner.offset() - 1, line, column); } line++; column = 0; break outer; default: buffer.append(c); } } parseBuffer(startOffset, startColumn); if (scanner.eof()) { addToken(TokenType.EOF, "", scanner.offset(), line, column); } }