@Test
public void testStringSlashes() throws IOException {
Token token;
Tokenizer tokenizer = new Tokenizer(new Context(" 'foo\\'bar' \"foo\\\"bar\""));
token = tokenizer.nextToken();
assertTrue(token instanceof StringExprToken);
assertEquals("foo'bar", ((StringExprToken) token).getValue());
token = tokenizer.nextToken();
assertTrue(token instanceof StringExprToken);
assertEquals("foo\"bar", ((StringExprToken) token).getValue());
}
@Test
public void testComplexOperators() throws IOException {
Tokenizer tokenizer = new Tokenizer(new Context("== >= <= === !== != && ||"));
assertTrue(tokenizer.nextToken() instanceof EqualExprToken);
assertTrue(tokenizer.nextToken() instanceof GreaterOrEqualExprToken);
assertTrue(tokenizer.nextToken() instanceof SmallerOrEqualToken);
assertTrue(tokenizer.nextToken() instanceof IdenticalExprToken);
assertTrue(tokenizer.nextToken() instanceof NotIdenticalExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanNotEqualExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanAndExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanOrExprToken);
}
@Test
public void testMacro() throws IOException {
Tokenizer tokenizer =
new Tokenizer(
new Context(
"__LINE__ __FILE__ __DIR__ __METHOD__ __FUNCTION__ __CLASS__ __NAMESPACE__ __TRAIT__"));
assertTrue(tokenizer.nextToken() instanceof LineMacroToken);
assertTrue(tokenizer.nextToken() instanceof FileMacroToken);
assertTrue(tokenizer.nextToken() instanceof DirMacroToken);
assertTrue(tokenizer.nextToken() instanceof MethodMacroToken);
assertTrue(tokenizer.nextToken() instanceof FunctionMacroToken);
assertTrue(tokenizer.nextToken() instanceof ClassMacroToken);
assertTrue(tokenizer.nextToken() instanceof NamespaceMacroToken);
assertTrue(tokenizer.nextToken() instanceof TraitMacroToken);
}
@Test
public void testComments() throws IOException {
Tokenizer tokenizer = new Tokenizer(new Context("/** FOO BAR \n\r100500 */"));
Token token = tokenizer.nextToken();
assertTrue(token instanceof CommentToken);
assertEquals(CommentToken.Kind.DOCTYPE, ((CommentToken) token).getKind());
assertEquals("FOO BAR\n100500", ((CommentToken) token).getComment());
// simple
tokenizer = new Tokenizer(new Context("// foobar \n $x"));
token = tokenizer.nextToken();
assertTrue(token instanceof CommentToken);
assertEquals(CommentToken.Kind.SIMPLE, ((CommentToken) token).getKind());
assertEquals(" foobar ", ((CommentToken) token).getComment());
assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
assertNull(tokenizer.nextToken());
tokenizer = new Tokenizer(new Context("# // foobar \n $x"));
token = tokenizer.nextToken();
assertTrue(token instanceof CommentToken);
assertEquals(CommentToken.Kind.SIMPLE, ((CommentToken) token).getKind());
assertEquals("// foobar ", ((CommentToken) token).getComment());
assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
assertNull(tokenizer.nextToken());
// block
tokenizer = new Tokenizer(new Context("/* foobar \n */"));
token = tokenizer.nextToken();
assertTrue(token instanceof CommentToken);
assertEquals(CommentToken.Kind.BLOCK, ((CommentToken) token).getKind());
}
@Test
public void testScalarTokens() throws IOException {
Token token;
Tokenizer tokenizer = new Tokenizer(new Context("10 3.3 'foo' '' \"xyz\" 0xCC true false"));
token = tokenizer.nextToken();
assertTrue(token instanceof IntegerExprToken);
assertEquals(10L, ((IntegerExprToken) token).getValue());
token = tokenizer.nextToken();
assertTrue(token instanceof DoubleExprToken);
assertEquals(3.3, ((DoubleExprToken) token).getValue(), 0.01);
token = tokenizer.nextToken();
assertTrue(token instanceof StringExprToken);
assertEquals(StringExprToken.Quote.SINGLE, ((StringExprToken) token).getQuote());
assertEquals("foo", ((StringExprToken) token).getValue());
token = tokenizer.nextToken();
assertTrue(token instanceof StringExprToken);
assertEquals(StringExprToken.Quote.SINGLE, ((StringExprToken) token).getQuote());
assertEquals("", ((StringExprToken) token).getValue());
token = tokenizer.nextToken();
assertTrue(token instanceof StringExprToken);
assertEquals(StringExprToken.Quote.DOUBLE, ((StringExprToken) token).getQuote());
assertEquals("xyz", ((StringExprToken) token).getValue());
token = tokenizer.nextToken();
assertTrue(token instanceof HexExprValue);
assertEquals(new BigInteger("CC", 16).longValue(), ((HexExprValue) token).getValue());
token = tokenizer.nextToken();
assertTrue(token instanceof BooleanExprToken);
assertEquals(true, ((BooleanExprToken) token).getValue());
token = tokenizer.nextToken();
assertTrue(token instanceof BooleanExprToken);
assertEquals(false, ((BooleanExprToken) token).getValue());
token = tokenizer.nextToken();
assertNull(token);
}
@Test
public void testHeredoc() throws IOException {
Tokenizer tokenizer = new Tokenizer(new Context("<<<DOC\n <foobar> \nDOC;\n"));
Token token = tokenizer.nextToken();
assertTrue(token instanceof StringExprToken);
assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
assertEquals(" <foobar> ", ((StringExprToken) token).getValue());
tokenizer = new Tokenizer(new Context("<<<\"DOC\"\n \\n<foobar> \nDOC;\n"));
token = tokenizer.nextToken();
assertTrue(token instanceof StringExprToken);
assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
assertEquals(" \n<foobar> ", ((StringExprToken) token).getValue());
tokenizer = new Tokenizer(new Context("<<<'DOC'\n \\n<foobar> \nDOC;\n"));
token = tokenizer.nextToken();
assertTrue(token instanceof StringExprToken);
assertEquals(StringExprToken.Quote.DOC, ((StringExprToken) token).getQuote());
assertEquals(" \\n<foobar> ", ((StringExprToken) token).getValue());
}
@Test
public void testParseError() throws IOException {
Throwable ex = null;
Tokenizer tokenizer = new Tokenizer(new Context(" 'foobar \n "));
try {
tokenizer.nextToken();
} catch (Throwable e) {
ex = e;
}
assertTrue(ex instanceof ParseException);
TraceInfo traceInfo = ((ParseException) ex).getTraceInfo();
assertNotNull(traceInfo);
assertNull(traceInfo.getContext().getFileName());
assertEquals(1, traceInfo.getStartLine());
assertEquals(1, traceInfo.getEndLine());
assertEquals(1, traceInfo.getStartPosition());
assertEquals(1, traceInfo.getEndPosition());
}
@Test
public void testSimpleOperators() throws IOException {
Tokenizer tokenizer = new Tokenizer(new Context("= + - / * % . and or new && || ! xor"));
assertTrue(tokenizer.nextToken() instanceof AssignExprToken);
assertTrue(tokenizer.nextToken() instanceof PlusExprToken);
assertTrue(tokenizer.nextToken() instanceof MinusExprToken);
assertTrue(tokenizer.nextToken() instanceof DivExprToken);
assertTrue(tokenizer.nextToken() instanceof MulExprToken);
assertTrue(tokenizer.nextToken() instanceof ModExprToken);
assertTrue(tokenizer.nextToken() instanceof ConcatExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanAnd2ExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanOr2ExprToken);
assertTrue(tokenizer.nextToken() instanceof NewExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanAndExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanOrExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanNotExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanXorExprToken);
}
@Test
public void testBraces() throws IOException {
Tokenizer tokenizer = new Tokenizer(new Context(" :: ->foobar('a', 1, 3.0);"));
assertTrue(tokenizer.nextToken() instanceof StaticAccessExprToken);
assertTrue(tokenizer.nextToken() instanceof DynamicAccessExprToken);
assertTrue(tokenizer.nextToken() instanceof NameToken);
assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
assertTrue(tokenizer.nextToken() instanceof StringExprToken);
assertTrue(tokenizer.nextToken() instanceof CommaToken);
assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
assertTrue(tokenizer.nextToken() instanceof CommaToken);
assertTrue(tokenizer.nextToken() instanceof DoubleExprToken);
assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
}
@Test
public void testVarVar() throws IOException {
Tokenizer tokenizer = new Tokenizer(new Context("$$foo $ $bar $$$foobar"));
assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
assertTrue(tokenizer.nextToken() instanceof VariableExprToken);
tokenizer = new Tokenizer(new Context("${'foo;6bar'}"));
assertTrue(tokenizer.nextToken() instanceof DollarExprToken);
assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
assertTrue(tokenizer.nextToken() instanceof StringExprToken);
assertTrue(tokenizer.nextToken() instanceof BraceExprToken);
}
@Test
public void testSimple() throws IOException {
Tokenizer tokenizer = new Tokenizer(new Context(""));
assertNull(tokenizer.nextToken());
assertEquals("", tokenizer.getCode());
tokenizer = new Tokenizer(new Context(" "));
assertNull(tokenizer.nextToken());
tokenizer = new Tokenizer(new Context(" "));
assertNull(tokenizer.nextToken());
tokenizer = new Tokenizer(new Context("\t"));
assertNull(tokenizer.nextToken());
tokenizer = new Tokenizer(new Context("\n"));
assertNull(tokenizer.nextToken());
tokenizer = new Tokenizer(new Context("\r"));
assertNull(tokenizer.nextToken());
}
@Test
public void testSplitNot() throws IOException {
Tokenizer tokenizer = new Tokenizer(new Context("!true"));
assertTrue(tokenizer.nextToken() instanceof BooleanNotExprToken);
assertTrue(tokenizer.nextToken() instanceof BooleanExprToken);
}
@Test
public void testStmt() throws IOException {
Tokenizer tokenizer =
new Tokenizer(
new Context(
"class function private public protected static final try catch for if foreach switch while "
+ "default return declare case do else elseif endif endfor endforeach endwhile endswitch "
+ "abstract use namespace finally extends implements global"));
assertTrue(tokenizer.nextToken() instanceof ClassStmtToken);
assertTrue(tokenizer.nextToken() instanceof FunctionStmtToken);
assertTrue(tokenizer.nextToken() instanceof PrivateStmtToken);
assertTrue(tokenizer.nextToken() instanceof PublicStmtToken);
assertTrue(tokenizer.nextToken() instanceof ProtectedStmtToken);
assertTrue(tokenizer.nextToken() instanceof StaticExprToken);
assertTrue(tokenizer.nextToken() instanceof FinalStmtToken);
assertTrue(tokenizer.nextToken() instanceof TryStmtToken);
assertTrue(tokenizer.nextToken() instanceof CatchStmtToken);
assertTrue(tokenizer.nextToken() instanceof ForStmtToken);
assertTrue(tokenizer.nextToken() instanceof IfStmtToken);
assertTrue(tokenizer.nextToken() instanceof ForeachStmtToken);
assertTrue(tokenizer.nextToken() instanceof SwitchStmtToken);
assertTrue(tokenizer.nextToken() instanceof WhileStmtToken);
assertTrue(tokenizer.nextToken() instanceof DefaultStmtToken);
assertTrue(tokenizer.nextToken() instanceof ReturnStmtToken);
assertTrue(tokenizer.nextToken() instanceof DeclareStmtToken);
assertTrue(tokenizer.nextToken() instanceof CaseStmtToken);
assertTrue(tokenizer.nextToken() instanceof DoStmtToken);
assertTrue(tokenizer.nextToken() instanceof ElseStmtToken);
assertTrue(tokenizer.nextToken() instanceof ElseIfStmtToken);
assertTrue(tokenizer.nextToken() instanceof EndifStmtToken);
assertTrue(tokenizer.nextToken() instanceof EndforStmtToken);
assertTrue(tokenizer.nextToken() instanceof EndforeachStmtToken);
assertTrue(tokenizer.nextToken() instanceof EndwhileStmtToken);
assertTrue(tokenizer.nextToken() instanceof EndswitchStmtToken);
assertTrue(tokenizer.nextToken() instanceof AbstractStmtToken);
assertTrue(tokenizer.nextToken() instanceof NamespaceUseStmtToken);
assertTrue(tokenizer.nextToken() instanceof NamespaceStmtToken);
assertTrue(tokenizer.nextToken() instanceof FinallyStmtToken);
assertTrue(tokenizer.nextToken() instanceof ExtendsStmtToken);
assertTrue(tokenizer.nextToken() instanceof ImplementsStmtToken);
assertTrue(tokenizer.nextToken() instanceof GlobalStmtToken);
}
@Test
public void testComplex() throws IOException {
Tokenizer tokenizer = new Tokenizer(new Context("0==10==='30';"));
assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
assertTrue(tokenizer.nextToken() instanceof EqualExprToken);
assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
assertTrue(tokenizer.nextToken() instanceof IdenticalExprToken);
assertTrue(tokenizer.nextToken() instanceof StringExprToken);
assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
tokenizer = new Tokenizer(new Context("F =; 20;"));
Token token;
assertTrue((token = tokenizer.nextToken()) instanceof NameToken);
assertEquals("F", ((NameToken) token).getName());
assertTrue(tokenizer.nextToken() instanceof AssignExprToken);
assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
assertTrue(tokenizer.nextToken() instanceof IntegerExprToken);
assertTrue(tokenizer.nextToken() instanceof SemicolonToken);
tokenizer = new Tokenizer(new Context("123foobar MAX_64Bit"));
token = tokenizer.nextToken();
assertTrue(token instanceof IntegerExprToken);
token = tokenizer.nextToken();
assertTrue(token instanceof NameToken);
assertEquals("foobar", token.getWord());
assertTrue(tokenizer.nextToken() instanceof NameToken);
assertNull(tokenizer.nextToken());
assertNull(tokenizer.nextToken());
}