static {
for (TokenType type : values()) {
if (type.getCharacter() != null) {
charMap.put(type.getCharacter(), type);
}
}
}
@Test
public void testSearchValueOf() {
List<TokenType> res = TokenType.searchValueOf("NUMBER_*");
assertEquals(2, res.size());
assertTrue(res.contains(TokenType.NUMBER_FLOAT));
assertTrue(res.contains(TokenType.NUMBER_INT));
res = TokenType.searchValueOf("START");
assertEquals(1, res.size());
assertTrue(res.contains(TokenType.START));
}
private void tokenExpected(TokenType type, String text) {
myLexer.nextToken();
if (type != myLexer.getTokenType()) {
throw new ParsingException(type.getName() + " expected but " + myLexer.getToken() + " found");
}
if (text != null && !text.equals(myLexer.getToken())) {
throw new ParsingException(text + " expected but " + myLexer.getToken() + " found");
}
}
/**
* Gets the text without the comments. For example for the string <code>{ // it's a comment</code>
* this method will return "{ ".
*
* @param aStart start of the text.
* @param anEnd end of the text.
* @return String for the line without comments (if exists).
*/
public synchronized String getUncommentedText(int aStart, int anEnd) {
readLock();
StringBuilder result = new StringBuilder();
Iterator<Token> iter = getTokens(aStart, anEnd);
while (iter.hasNext()) {
Token t = iter.next();
if (!TokenType.isComment(t)) {
result.append(t.getText(this));
}
}
readUnlock();
return result.toString();
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
Token other = (Token) obj;
if (!tokenType.equals(other.tokenType)) return false;
if (value == null) {
if (other.value != null) return false;
} else if (!value.equals(other.value)) return false;
return true;
}
public String toString() {
return Type.toString() + Value;
}
public Terminal(String termName) {
this.tokenType = TokenType.valueOf(termName);
}
@Override
public int hashCode() {
return tokenType.hashCode();
}
@Override
public String toString() {
return tokenType.toString();
}
@Override
public String toString() {
return String.format("(%s '%s')", type.name(), data);
}
public boolean hasType(TokenType t) {
return t.equals(getType());
}
public int getLastTokenPositionIndex(TokenType tokenType) {
return Integer.parseInt(getProperty("lastTokenPositionIndex-" + tokenType.name(), "0"));
}
public void setTypesCountStdDev(TokenType tokenType, float mean) {
setProperty("typesCountStdDev-" + tokenType.name(), String.valueOf(mean));
}
public void setTypesCount(TokenType tokenType, int totalTypes) {
setProperty("typesCount-" + tokenType.name(), String.valueOf(totalTypes));
}
public void setLastTokenPositionIndex(TokenType tokenType, int lastPosition) {
setProperty("lastTokenPositionIndex-" + tokenType.name(), String.valueOf(lastPosition));
}
public void setLastTokenOffsetIndex(TokenType tokenType, int lastOffset) {
setProperty("lastTokenStartOffset-" + tokenType.name(), String.valueOf(lastOffset));
}
public float getTypesCountStdDev(TokenType tokenType) {
return Float.parseFloat(getProperty("typesCountStdDev-" + tokenType.name(), "0"));
}
