static { for (TokenType type : values()) { if (type.getCharacter() != null) { charMap.put(type.getCharacter(), type); } } }
@Test public void testSearchValueOf() { List<TokenType> res = TokenType.searchValueOf("NUMBER_*"); assertEquals(2, res.size()); assertTrue(res.contains(TokenType.NUMBER_FLOAT)); assertTrue(res.contains(TokenType.NUMBER_INT)); res = TokenType.searchValueOf("START"); assertEquals(1, res.size()); assertTrue(res.contains(TokenType.START)); }
private void tokenExpected(TokenType type, String text) { myLexer.nextToken(); if (type != myLexer.getTokenType()) { throw new ParsingException(type.getName() + " expected but " + myLexer.getToken() + " found"); } if (text != null && !text.equals(myLexer.getToken())) { throw new ParsingException(text + " expected but " + myLexer.getToken() + " found"); } }
/** * Gets the text without the comments. For example for the string <code>{ // it's a comment</code> * this method will return "{ ". * * @param aStart start of the text. * @param anEnd end of the text. * @return String for the line without comments (if exists). */ public synchronized String getUncommentedText(int aStart, int anEnd) { readLock(); StringBuilder result = new StringBuilder(); Iterator<Token> iter = getTokens(aStart, anEnd); while (iter.hasNext()) { Token t = iter.next(); if (!TokenType.isComment(t)) { result.append(t.getText(this)); } } readUnlock(); return result.toString(); }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; Token other = (Token) obj; if (!tokenType.equals(other.tokenType)) return false; if (value == null) { if (other.value != null) return false; } else if (!value.equals(other.value)) return false; return true; }
public String toString() { return Type.toString() + Value; }
public Terminal(String termName) { this.tokenType = TokenType.valueOf(termName); }
@Override public int hashCode() { return tokenType.hashCode(); }
@Override public String toString() { return tokenType.toString(); }
@Override public String toString() { return String.format("(%s '%s')", type.name(), data); }
public boolean hasType(TokenType t) { return t.equals(getType()); }
public int getLastTokenPositionIndex(TokenType tokenType) { return Integer.parseInt(getProperty("lastTokenPositionIndex-" + tokenType.name(), "0")); }
public void setTypesCountStdDev(TokenType tokenType, float mean) { setProperty("typesCountStdDev-" + tokenType.name(), String.valueOf(mean)); }
public void setTypesCount(TokenType tokenType, int totalTypes) { setProperty("typesCount-" + tokenType.name(), String.valueOf(totalTypes)); }
public void setLastTokenPositionIndex(TokenType tokenType, int lastPosition) { setProperty("lastTokenPositionIndex-" + tokenType.name(), String.valueOf(lastPosition)); }
public void setLastTokenOffsetIndex(TokenType tokenType, int lastOffset) { setProperty("lastTokenStartOffset-" + tokenType.name(), String.valueOf(lastOffset)); }
public float getTypesCountStdDev(TokenType tokenType) { return Float.parseFloat(getProperty("typesCountStdDev-" + tokenType.name(), "0")); }