@Test
public void testgetNextToken() {
Token token = null;
Token comparisonToken = null;
do {
comparisonToken = list.remove(0);
token = this.lexer.getNextToken();
assertEquals(comparisonToken.getValue(), token.getValue());
assertEquals(comparisonToken.getTokenType(), token.getTokenType());
if (token.getTokenType().equals(TokenType.NUM)) {
NumToken comparisonNumToken =
new NumTokenImpl(comparisonToken.getValue(), null, null, null);
NumToken numToken = new NumTokenImpl(token.getValue(), null, null, null);
assertEquals(comparisonNumToken.getLongValue(), numToken.getLongValue());
}
} while (token.getTokenType() != TokenType.EOF);
}
/**
* Test for tokenizing of num types
*
* @throws UnsupportedEncodingException : UTF-8 encoding not supported
*/
@Test
public void simpleTokenizingOfNumTypesTest() throws UnsupportedEncodingException {
String simpleNumString =
Constants.LONGSTRING1
+ " "
+ Constants.LONGSTRING2
+ " "
+ Constants.LONGSTRING3
+ " "
+ Constants.LONGSTRING4
+ " "
+ Constants.LONGSTRING5
+ " "
+ Constants.LONGSTRING6
+ " "
+ Constants.LONGSTRING7
+ " "
+ Constants.LONGSTRING8
+ " "
+ Constants.LONGSTRING9;
this.lexer.setSourceStream(new ByteArrayInputStream(simpleNumString.getBytes("UTF-8")));
NumToken token = (NumToken) this.lexer.getNextToken();
assertEquals(Constants.LONGSTRING1, token.getValue());
assertEquals(TokenType.NUM, token.getTokenType());
assertEquals(1, token.getLine().intValue());
assertEquals(1, token.getColumn().intValue());
assertTrue(token.getLongValue() == Long.valueOf(Constants.LONGSTRING1).longValue());
token = (NumToken) this.lexer.getNextToken();
assertEquals(Constants.LONGSTRING2, token.getValue());
assertEquals(TokenType.NUM, token.getTokenType());
assertEquals(1, token.getLine().intValue());
assertEquals(5, token.getColumn().intValue());
assertTrue(token.getLongValue() == Long.valueOf("12300").longValue());
token = (NumToken) this.lexer.getNextToken();
assertEquals(Constants.LONGSTRING3, token.getValue());
assertEquals(TokenType.NUM, token.getTokenType());
assertEquals(1, token.getLine().intValue());
assertEquals(11, token.getColumn().intValue());
assertTrue(token.getLongValue() == Long.valueOf("12300").longValue());
token = (NumToken) this.lexer.getNextToken();
assertEquals(Constants.LONGSTRING4, token.getValue());
assertEquals(TokenType.NUM, token.getTokenType());
assertEquals(1, token.getLine().intValue());
assertEquals(17, token.getColumn().intValue());
assertEquals(Constants.LONGSTRING4, token.getValue());
token = (NumToken) this.lexer.getNextToken();
assertEquals(Constants.LONGSTRING5, token.getValue());
assertEquals(TokenType.NUM, token.getTokenType());
assertEquals(1, token.getLine().intValue());
assertEquals(24, token.getColumn().intValue());
assertEquals(Constants.LONGSTRING5, token.getValue());
Token normalToken = this.lexer.getNextToken();
assertEquals(Constants.MINUSSTRING, normalToken.getValue());
assertEquals(TokenType.MINUS, normalToken.getTokenType());
assertEquals(1, normalToken.getLine().intValue());
assertEquals(31, normalToken.getColumn().intValue());
token = (NumToken) this.lexer.getNextToken();
assertEquals(Constants.LONGSTRING6.substring(1), token.getValue());
assertEquals(TokenType.NUM, token.getTokenType());
assertEquals(1, token.getLine().intValue());
assertEquals(32, token.getColumn().intValue());
assertTrue(
token.getLongValue() == Long.valueOf(Constants.LONGSTRING6.substring(1)).longValue());
normalToken = this.lexer.getNextToken();
assertEquals(Constants.MINUSSTRING, normalToken.getValue());
assertEquals(TokenType.MINUS, normalToken.getTokenType());
assertEquals(1, normalToken.getLine().intValue());
assertEquals(36, normalToken.getColumn().intValue());
token = (NumToken) this.lexer.getNextToken();
assertEquals(Constants.LONGSTRING7.substring(1), token.getValue());
assertEquals(TokenType.NUM, token.getTokenType());
assertEquals(1, token.getLine().intValue());
assertEquals(37, token.getColumn().intValue());
assertTrue(token.getLongValue() == Long.valueOf("12300").longValue());
normalToken = this.lexer.getNextToken();
assertEquals(Constants.MINUSSTRING, normalToken.getValue());
assertEquals(TokenType.MINUS, normalToken.getTokenType());
assertEquals(1, normalToken.getLine().intValue());
assertEquals(43, normalToken.getColumn().intValue());
token = (NumToken) this.lexer.getNextToken();
assertEquals(Constants.LONGSTRING8.substring(1), token.getValue());
assertEquals(TokenType.NUM, token.getTokenType());
assertEquals(1, token.getLine().intValue());
assertEquals(44, token.getColumn().intValue());
assertTrue(token.getLongValue() == Long.valueOf("12300").longValue());
normalToken = this.lexer.getNextToken();
assertEquals(Constants.MINUSSTRING, normalToken.getValue());
assertEquals(TokenType.MINUS, normalToken.getTokenType());
assertEquals(1, normalToken.getLine().intValue());
assertEquals(50, normalToken.getColumn().intValue());
token = (NumToken) this.lexer.getNextToken();
assertEquals(Constants.LONGSTRING9.substring(1), token.getValue());
assertEquals(TokenType.NUM, token.getTokenType());
assertEquals(1, token.getLine().intValue());
assertEquals(51, token.getColumn().intValue());
assertEquals(Constants.LONGSTRING9.substring(1), token.getValue());
}
