@Test
  public void testgetNextToken() {
    Token token = null;
    Token comparisonToken = null;

    do {

      comparisonToken = list.remove(0);
      token = this.lexer.getNextToken();

      assertEquals(comparisonToken.getValue(), token.getValue());
      assertEquals(comparisonToken.getTokenType(), token.getTokenType());

      if (token.getTokenType().equals(TokenType.NUM)) {

        NumToken comparisonNumToken =
            new NumTokenImpl(comparisonToken.getValue(), null, null, null);
        NumToken numToken = new NumTokenImpl(token.getValue(), null, null, null);
        assertEquals(comparisonNumToken.getLongValue(), numToken.getLongValue());
      }

    } while (token.getTokenType() != TokenType.EOF);
  }
예제 #2
0
  /**
   * Test for tokenizing of num types
   *
   * @throws UnsupportedEncodingException : UTF-8 encoding not supported
   */
  @Test
  public void simpleTokenizingOfNumTypesTest() throws UnsupportedEncodingException {
    String simpleNumString =
        Constants.LONGSTRING1
            + " "
            + Constants.LONGSTRING2
            + " "
            + Constants.LONGSTRING3
            + " "
            + Constants.LONGSTRING4
            + " "
            + Constants.LONGSTRING5
            + " "
            + Constants.LONGSTRING6
            + " "
            + Constants.LONGSTRING7
            + " "
            + Constants.LONGSTRING8
            + " "
            + Constants.LONGSTRING9;

    this.lexer.setSourceStream(new ByteArrayInputStream(simpleNumString.getBytes("UTF-8")));

    NumToken token = (NumToken) this.lexer.getNextToken();
    assertEquals(Constants.LONGSTRING1, token.getValue());
    assertEquals(TokenType.NUM, token.getTokenType());
    assertEquals(1, token.getLine().intValue());
    assertEquals(1, token.getColumn().intValue());
    assertTrue(token.getLongValue() == Long.valueOf(Constants.LONGSTRING1).longValue());

    token = (NumToken) this.lexer.getNextToken();
    assertEquals(Constants.LONGSTRING2, token.getValue());
    assertEquals(TokenType.NUM, token.getTokenType());
    assertEquals(1, token.getLine().intValue());
    assertEquals(5, token.getColumn().intValue());
    assertTrue(token.getLongValue() == Long.valueOf("12300").longValue());

    token = (NumToken) this.lexer.getNextToken();
    assertEquals(Constants.LONGSTRING3, token.getValue());
    assertEquals(TokenType.NUM, token.getTokenType());
    assertEquals(1, token.getLine().intValue());
    assertEquals(11, token.getColumn().intValue());
    assertTrue(token.getLongValue() == Long.valueOf("12300").longValue());

    token = (NumToken) this.lexer.getNextToken();
    assertEquals(Constants.LONGSTRING4, token.getValue());
    assertEquals(TokenType.NUM, token.getTokenType());
    assertEquals(1, token.getLine().intValue());
    assertEquals(17, token.getColumn().intValue());
    assertEquals(Constants.LONGSTRING4, token.getValue());

    token = (NumToken) this.lexer.getNextToken();
    assertEquals(Constants.LONGSTRING5, token.getValue());
    assertEquals(TokenType.NUM, token.getTokenType());
    assertEquals(1, token.getLine().intValue());
    assertEquals(24, token.getColumn().intValue());
    assertEquals(Constants.LONGSTRING5, token.getValue());

    Token normalToken = this.lexer.getNextToken();
    assertEquals(Constants.MINUSSTRING, normalToken.getValue());
    assertEquals(TokenType.MINUS, normalToken.getTokenType());
    assertEquals(1, normalToken.getLine().intValue());
    assertEquals(31, normalToken.getColumn().intValue());

    token = (NumToken) this.lexer.getNextToken();
    assertEquals(Constants.LONGSTRING6.substring(1), token.getValue());
    assertEquals(TokenType.NUM, token.getTokenType());
    assertEquals(1, token.getLine().intValue());
    assertEquals(32, token.getColumn().intValue());
    assertTrue(
        token.getLongValue() == Long.valueOf(Constants.LONGSTRING6.substring(1)).longValue());

    normalToken = this.lexer.getNextToken();
    assertEquals(Constants.MINUSSTRING, normalToken.getValue());
    assertEquals(TokenType.MINUS, normalToken.getTokenType());
    assertEquals(1, normalToken.getLine().intValue());
    assertEquals(36, normalToken.getColumn().intValue());

    token = (NumToken) this.lexer.getNextToken();
    assertEquals(Constants.LONGSTRING7.substring(1), token.getValue());
    assertEquals(TokenType.NUM, token.getTokenType());
    assertEquals(1, token.getLine().intValue());
    assertEquals(37, token.getColumn().intValue());
    assertTrue(token.getLongValue() == Long.valueOf("12300").longValue());

    normalToken = this.lexer.getNextToken();
    assertEquals(Constants.MINUSSTRING, normalToken.getValue());
    assertEquals(TokenType.MINUS, normalToken.getTokenType());
    assertEquals(1, normalToken.getLine().intValue());
    assertEquals(43, normalToken.getColumn().intValue());

    token = (NumToken) this.lexer.getNextToken();
    assertEquals(Constants.LONGSTRING8.substring(1), token.getValue());
    assertEquals(TokenType.NUM, token.getTokenType());
    assertEquals(1, token.getLine().intValue());
    assertEquals(44, token.getColumn().intValue());
    assertTrue(token.getLongValue() == Long.valueOf("12300").longValue());

    normalToken = this.lexer.getNextToken();
    assertEquals(Constants.MINUSSTRING, normalToken.getValue());
    assertEquals(TokenType.MINUS, normalToken.getTokenType());
    assertEquals(1, normalToken.getLine().intValue());
    assertEquals(50, normalToken.getColumn().intValue());

    token = (NumToken) this.lexer.getNextToken();
    assertEquals(Constants.LONGSTRING9.substring(1), token.getValue());
    assertEquals(TokenType.NUM, token.getTokenType());
    assertEquals(1, token.getLine().intValue());
    assertEquals(51, token.getColumn().intValue());
    assertEquals(Constants.LONGSTRING9.substring(1), token.getValue());
  }