@Test public void NonWordBoundaryTest() throws Exception { String input = "\\B"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.NonWordBoundary)); }
private int getElementOffset(CommonTree tree) { switch (tree.getType()) { case ANTLRParser.MODE: case ANTLRParser.ASSIGN: case ANTLRParser.RULE: if (tree.getChildCount() > 0 && tree.getChild(0) instanceof CommonTree) { CommonTree child = (CommonTree) tree.getChild(0); if (child.getToken() instanceof CommonToken) { CommonToken token = (CommonToken) child.getToken(); return token.getStartIndex(); } } break; case ANTLRParser.ID: break; default: throw new UnsupportedOperationException(); } if (tree.getToken() instanceof CommonToken) { return ((CommonToken) tree.getToken()).getStartIndex(); } return 0; }
@Test public void NotDecimalDigitTest() throws Exception { String input = "\\D"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.NotDecimalDigit)); }
@Test public void OneDataUnitTest() throws Exception { String input = "\\C"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.OneDataUnit)); }
@Test public void EndOfSubjectTest() throws Exception { String input = "\\z"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.EndOfSubject)); }
private String getCoreSQL(RuleReturnScope parsedSQL) { final CommonTree ast = (CommonTree) parsedSQL.getTree(); final CommonToken nameToken = (CommonToken) ((CommonTree) ast.getChild(1)).getToken(); final CharStream inputStream = nameToken.getInputStream(); final CommonToken stopToken = (CommonToken) parsedSQL.getStop(); return inputStream.substring(nameToken.getStartIndex(), stopToken.getStopIndex()); }
@Test public void NotWordCharTest() throws Exception { String input = "\\W"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.NotWordChar)); }
@Test public void ExtendedUnicodeCharTest() throws Exception { String input = "\\X"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.ExtendedUnicodeChar)); }
@Test public void NotVerticalWhiteSpaceTest() throws Exception { String input = "\\V"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.NotVerticalWhiteSpace)); }
@Test public void PreviousMatchInSubjectTest() throws Exception { String input = "\\G"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.PreviousMatchInSubject)); }
@Test public void ResetStartMatchTest() throws Exception { String input = "\\K"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.ResetStartMatch)); }
@Test public void NewLineSequenceTest() throws Exception { String input = "\\R"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.NewLineSequence)); }
public void assertNonSmoking(final CharSequence input) throws Exception { final String string = input.toString(); final List<CommonToken> tokenList = CollectionLiterals.<CommonToken>newArrayList(); { final Lexer lexer = this.lexerProvider.get(); ANTLRStringStream _aNTLRStringStream = new ANTLRStringStream(string); lexer.setCharStream(_aNTLRStringStream); Token token = lexer.nextToken(); boolean _notEquals = (!Objects.equal(token, Token.EOF_TOKEN)); boolean _while = _notEquals; while (_while) { { tokenList.add(((CommonToken) token)); Token _nextToken = lexer.nextToken(); token = _nextToken; } boolean _notEquals_1 = (!Objects.equal(token, Token.EOF_TOKEN)); _while = _notEquals_1; } } for (final CommonToken token : tokenList) { { final int start = token.getStartIndex(); String _text = token.getText(); final int length = _text.length(); String _substring = string.substring(0, start); String _substring_1 = string.substring((start + length)); String _plus = (_substring + _substring_1); this.processFile(_plus); } } }
private int getStringOrCommentIndent(int offset) { CommonToken tokenContainingOffset = getTokenStrictlyContainingOffset(offset); CommonToken token = getStartOfStringToken(tokenContainingOffset); if (token != null) { int type = token.getType(); int start = token.getCharPositionInLine(); if (token.getStartIndex() < offset) { switch (type) { case STRING_LITERAL: case STRING_START: case ASTRING_LITERAL: return start + 1; case STRING_MID: case STRING_END: case MULTI_COMMENT: return start + 1; // uncomment to get a bigger indent // return start+3; case VERBATIM_STRING: case AVERBATIM_STRING: return start + 3; } } } return -1; }
@Test public void VUCTest() throws Exception { String input = "V"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.VUC)); assertThat(token.getText(), is(input)); }
@Test public void AmpersandTest() throws Exception { String input = "&"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.Ampersand)); assertThat(token.getText(), is(input)); }
@Test public void UnderscoreTest() throws Exception { String input = "_"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.Underscore)); assertThat(token.getText(), is(input)); }
@Test public void SingleQuoteTest() throws Exception { String input = "'"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.SingleQuote)); assertThat(token.getText(), is(input)); }
@Test public void GreaterThanTest() throws Exception { String input = ">"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.GreaterThan)); assertThat(token.getText(), is(input)); }
@Test public void CloseParenTest() throws Exception { String input = ")"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.CloseParen)); assertThat(token.getText(), is(input)); }
@Test public void NamedReferenceStartKTest() throws Exception { String input = "\\k"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.NamedReferenceStartK)); assertThat(token.getText(), is(input)); }
@Test public void POSIXNegatedNamedSetTest() throws Exception { for (String name : PCRELexer.namedSet) { CommonToken token = getToken(String.format("[[:^%s:]]", name)); assertThat(token.getType(), is(PCRELexer.POSIXNegatedNamedSet)); } }
@Test public void BackslashTest() throws Exception { CommonToken token = getToken("\\"); assertThat(token.getType(), is(PCRELexer.Backslash)); assertThat(token.getText(), is("\\")); }
@Test public void OpenBraceTest() throws Exception { String input = "{"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.OpenBrace)); assertThat(token.getText(), is(input)); }
@Test public void QuestionMarkTest() throws Exception { String input = "?"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.QuestionMark)); assertThat(token.getText(), is(input)); }
@Test public void SubroutineOrNamedReferenceStartGTest() throws Exception { String input = "\\g"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.SubroutineOrNamedReferenceStartG)); assertThat(token.getText(), is(input)); }
private int getTokenTypeOfCharacterAtOffset(int offset) { int tokenIndex = getTokenIndexAtCharacter(tokens, offset); if (tokenIndex >= 0) { CommonToken token = tokens.get(tokenIndex); return token.getType(); } return -1; }
@Test public void CharacterClassEndTest() throws Exception { String input = "]"; CommonToken token = getToken(input); assertThat(token.getType(), is(PCRELexer.CharacterClassEnd)); assertThat(token.getText(), is(input)); }
@Test public void OtherCharTest() throws Exception { List<CommonToken> tokens = getTokens("~ ` @ % \" ; / \u01FF \uABCD"); for (CommonToken token : tokens) { assertThat(token.getType(), is(PCRELexer.OtherChar)); } }
@Test public void CharWithoutPropertyTest() throws Exception { for (String property : PCRELexer.propertySet) { CommonToken token = getToken(String.format("\\P{%s}", property)); assertThat(token.getType(), is(PCRELexer.CharWithoutProperty)); } }