@Test
public void testCss_happyPath_simpleSelector() {
String code = "body { padding: 0; }";
Segment segment = new Segment(code.toCharArray(), 0, code.length());
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, CSS_PREV_TOKEN_TYPE, 0);
Assert.assertTrue(token.is(TokenTypes.DATA_TYPE, "body"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " "));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.SEPARATOR, "{"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " "));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.RESERVED_WORD, "padding"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.OPERATOR, ":"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " "));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.LITERAL_NUMBER_DECIMAL_INT, "0"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.OPERATOR, ";"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " "));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.SEPARATOR, "}"));
}
@Test
public void testSeparators() {
String code = "( ) [ ] { }";
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
String[] separators = code.split(" +");
for (int i = 0; i < separators.length; i++) {
Assert.assertEquals(separators[i], token.getLexeme());
Assert.assertEquals(TokenTypes.SEPARATOR, token.getType());
// Just one extra test here
Assert.assertTrue(token.isSingleChar(TokenTypes.SEPARATOR, separators[i].charAt(0)));
if (i < separators.length - 1) {
token = token.getNextToken();
Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace());
Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " "));
}
token = token.getNextToken();
}
Assert.assertTrue(token.getType() == TokenTypes.NULL);
}
@Test
public void testHexLiterals() {
String code =
"0x1 0xf 0x333333333333 0X1 0Xf 0X33333333333 0xF 0XF "
+ "0x1 0xf 0x333333333333 0X1 0Xf 0X33333333333 0xF 0XF";
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
String[] keywords = code.split(" +");
for (int i = 0; i < keywords.length; i++) {
Assert.assertEquals(keywords[i], token.getLexeme());
Assert.assertEquals(
"Invalid hex literal: " + token, TokenTypes.LITERAL_NUMBER_HEXADECIMAL, token.getType());
if (i < keywords.length - 1) {
token = token.getNextToken();
Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace());
Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " "));
}
token = token.getNextToken();
}
Assert.assertTrue(token.getType() == TokenTypes.NULL);
}
@Test
public void testOperators() {
String assignmentOperators = "+ - <= ^ ++ < * >= % -- > / != ? >> ! & == : >> ~ | &&";
String nonAssignmentOperators = "= -= *= /= |= &= ^= += %= <<= >>=";
String code = assignmentOperators + " " + nonAssignmentOperators;
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
String[] keywords = code.split(" +");
for (int i = 0; i < keywords.length; i++) {
Assert.assertEquals(keywords[i], token.getLexeme());
Assert.assertEquals("Not an operator: " + token, TokenTypes.OPERATOR, token.getType());
if (i < keywords.length - 1) {
token = token.getNextToken();
Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace());
Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " "));
}
token = token.getNextToken();
}
Assert.assertTrue(token.getType() == TokenTypes.NULL);
}
@Test
public void testMultiLineComments_URL() {
String[] mlcLiterals = {
"/* Hello world http://www.sas.com */",
};
for (String code : mlcLiterals) {
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
Assert.assertEquals(TokenTypes.COMMENT_MULTILINE, token.getType());
token = token.getNextToken();
Assert.assertTrue(token.isHyperlink());
Assert.assertEquals(TokenTypes.COMMENT_MULTILINE, token.getType());
Assert.assertEquals("http://www.sas.com", token.getLexeme());
token = token.getNextToken();
Assert.assertEquals(TokenTypes.COMMENT_MULTILINE, token.getType());
Assert.assertEquals(" */", token.getLexeme());
}
}
@Test
public void testKeywords() {
String code = "function functionend section sectionend subsection subsectionend";
// Conditional compilation
code += " !ifdef !ifndef !if !else !endif !macro !macroend";
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
String[] keywords = code.split(" +");
for (int i = 0; i < keywords.length; i++) {
Assert.assertEquals(keywords[i], token.getLexeme());
Assert.assertEquals("Not a keyword: " + token, TokenTypes.RESERVED_WORD, token.getType());
if (i < keywords.length - 1) {
token = token.getNextToken();
Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace());
Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " "));
}
token = token.getNextToken();
}
Assert.assertTrue(token.getType() == TokenTypes.NULL);
}
@Test
public void testStandardFunctions() {
String[] functions = {
"abort",
"call",
"clearerrors",
"goto",
"ifabort",
"iferrors",
"iffileexists",
"ifrebootflag",
"intcmp",
"intcmpu",
"iswindow",
"messagebox",
"reboot",
"return",
"quit",
"seterrors",
"strcmp",
"strcmps",
};
for (String code : functions) {
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
Assert.assertEquals(TokenTypes.FUNCTION, token.getType());
}
}
@Test
public void testCss_propertyValue_function() {
String code = "background-image: url(\"test.png\");";
Segment segment = new Segment(code.toCharArray(), 0, code.length());
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, CSSTokenMaker.INTERNAL_CSS_PROPERTY, 0);
Assert.assertTrue(token.is(TokenTypes.RESERVED_WORD, "background-image"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.OPERATOR, ":"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " "));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.FUNCTION, "url"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.SEPARATOR, "("));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, "\"test.png\""));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.SEPARATOR, ")"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.OPERATOR, ";"));
code = "background-image: url('test.png');";
segment = new Segment(code.toCharArray(), 0, code.length());
tm = createTokenMaker();
token = tm.getTokenList(segment, CSSTokenMaker.INTERNAL_CSS_PROPERTY, 0);
Assert.assertTrue(token.is(TokenTypes.RESERVED_WORD, "background-image"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.OPERATOR, ":"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " "));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.FUNCTION, "url"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.SEPARATOR, "("));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, "'test.png'"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.SEPARATOR, ")"));
token = token.getNextToken();
Assert.assertTrue(token.is(TokenTypes.OPERATOR, ";"));
}
@Test
public void testCss_id() {
String code = "#mainContent";
Segment segment = new Segment(code.toCharArray(), 0, code.length());
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, CSS_PREV_TOKEN_TYPE, 0);
Assert.assertTrue(token.is(TokenTypes.VARIABLE, "#mainContent"));
}
@Test
public void testCss_comment() {
String[] commentLiterals = {
"/* Hello world */",
};
for (String code : commentLiterals) {
Segment segment = new Segment(code.toCharArray(), 0, code.length());
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, CSS_PREV_TOKEN_TYPE, 0);
Assert.assertEquals(TokenTypes.COMMENT_MULTILINE, token.getType());
}
}
@Test
public void testEolComments() {
String[] eolCommentLiterals = {
"# Hello world", "; Hello world",
};
for (String code : eolCommentLiterals) {
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType());
}
}
@Test
public void testMultiLineComments() {
String[] mlcLiterals = {
"/* Hello world */",
};
for (String code : mlcLiterals) {
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
Assert.assertEquals(TokenTypes.COMMENT_MULTILINE, token.getType());
}
}
@Test
public void testCharLiterals() {
String[] chars = {
"''", "'hi'", "'\"\"'", "'$\'escaped single quote'",
};
for (String code : chars) {
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
Assert.assertEquals(
"Invalid char literal: " + token, TokenTypes.LITERAL_CHAR, token.getType());
}
}
@Test
public void testStringLiterals() {
String[] stringLiterals = {
"\"\"", "\"hi\"", "\"$\\\"escaped double quote\"",
};
for (String code : stringLiterals) {
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
Assert.assertEquals(
"Not a string: " + token, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, token.getType());
}
}
@Test
public void testCss_comment_URL() {
String code = "/* Hello world http://www.google.com */";
Segment segment = new Segment(code.toCharArray(), 0, code.length());
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, CSS_PREV_TOKEN_TYPE, 0);
Assert.assertFalse(token.isHyperlink());
Assert.assertTrue(token.is(TokenTypes.COMMENT_MULTILINE, "/* Hello world "));
token = token.getNextToken();
Assert.assertTrue(token.isHyperlink());
Assert.assertTrue(token.is(TokenTypes.COMMENT_MULTILINE, "http://www.google.com"));
token = token.getNextToken();
Assert.assertFalse(token.isHyperlink());
Assert.assertTrue(token.is(TokenTypes.COMMENT_MULTILINE, " */"));
}
@Test
public void testEolComments_URL() {
String[] eolCommentLiterals = {
"# Hello world http://www.sas.com", "; Hello world http://www.sas.com",
};
for (String code : eolCommentLiterals) {
Segment segment = createSegment(code);
TokenMaker tm = createTokenMaker();
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0);
Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType());
token = token.getNextToken();
Assert.assertTrue(token.isHyperlink());
Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType());
Assert.assertEquals("http://www.sas.com", token.getLexeme());
}
}