private Token newOneToken(String image, int column) {
Token token = new Token();
token.image = image;
token.beginLine = line;
token.beginColumn = column;
token.endLine = line;
token.endColumn = column;
token.kind = -1;
token.next = null;
return token;
}
/**
* read next token image string in current line
*
* @return null if reach end of file and no any valid string
*/
protected String getNextImage() throws ParserException {
String ret = "";
char lastChar = 0;
char c;
boolean error = false;
boolean first = true;
int curColumn = column + 1;
int max = strLine.length();
while (column < max && !error) // one token always in one line
{
c = strLine.charAt(column);
if (Character.isWhitespace(c)) {
if (!ret.isEmpty() && ret.indexOf(singleQuote) > 0 && !RegExp.is_number(ret)) {
column++;
continue;
} else {
break;
}
}
// escape identifier
if (backSlash == c) {
// while until whitespace or line feed or return
while (column < max && !Character.isWhitespace(c)) {
ret += strLine.charAt(column);
column++;
}
break;
}
if (!first
&& ((specialChar.indexOf(lastChar) < 0 && specialChar.indexOf(c) >= 0)
|| (specialChar.indexOf(c) < 0 && specialChar.indexOf(lastChar) >= 0))) {
if (column >= max) {
break;
}
if (Character.isDigit(lastChar)
&& c == '.'
&& Character.isDigit(strLine.charAt(column + 1))) {
// float point
ret += c;
column++;
c = strLine.charAt(column);
} else {
break; // exit loop when character change between specialChar and letter&digit
}
}
if (!first
&& Character.isDigit(lastChar)
&& !Character.isDigit(c)
&& c != '.'
&& c != singleQuote
&& !ret.startsWith("`")) {
String tmp = ret + c;
// divide some string(such as 10ns) into two token
if (!RegExp.is_IDENTIFIER(tmp) && !RegExp.is_number(tmp)) break;
}
// allow continuous specialChar
if (!first && specialChar.indexOf(c) >= 0 && specialChar.indexOf(lastChar) >= 0) {
// &&&
// ===, !==
// >=, <=, ==, !=
// ~&, &&
// ~|, ||
// ^~, ~^
// <<, >>
// =>, *>, ->
if ((column < max - 2)
&& (c == '&')
&& (strLine.charAt(column + 1) == '&')
&& (lastChar == '&')) {
ret += c;
ret += strLine.charAt(column + 1);
column += 2;
} else if ((column < max - 2)
&& (c == '=')
&& (strLine.charAt(column + 1) == '=')
&& ((lastChar == '!') || (lastChar == '='))) {
ret += c;
ret += strLine.charAt(column + 1);
column += 2;
} else if ((c == '='
&& (lastChar == '!' || lastChar == '>' || lastChar == '<' || lastChar == '='))
|| (c == '&' && (lastChar == '~' || lastChar == '&'))
|| (c == '|' && (lastChar == '~' || lastChar == '|'))
|| (c == '^' && lastChar == '~')
|| (c == '~' && lastChar == '^')
|| (c == '>' && lastChar == '>')
|| (c == '<' && lastChar == '<')
|| (c == '>' && (lastChar == '=' || lastChar == '*' || lastChar == '-'))) {
ret += c;
column++;
}
break;
}
// string
if (doubleQuote == c) {
ret += c;
column++;
while (column < max) {
char c1 = strLine.charAt(column);
ret += c1;
if (c1 == doubleQuote) {
column++;
if (column >= max) {
break;
}
c1 = strLine.charAt(column);
if (c1 != doubleQuote) {
break; // double quote in a string must a pair put together
}
ret += c1;
}
column++;
}
if (column >= max) {
// error = true; //TODO can double quote be next line?
}
break; // always quit loop on double quote
}
first = false;
ret += c;
lastChar = c;
column++;
}
if (error) {
throw new ParserException(newOneToken(ret, curColumn));
}
if (ret.isEmpty()) return null;
// check macro define
if (parser != null && ret.startsWith("`")) {
int index = -1;
for (int i = 0; i < CompilerDirectives.cdStrings.length; i++) {
if (ret.equals(CompilerDirectives.cdStrings[i])) {
index = i;
break;
}
}
if (index < 0) {
// extract the content of macro
ret = ret.substring(1);
ASTNode node = parser.getMacroContent(ret);
if (node == null) {
throw new SymbolNotFoundException(newOneToken(ret, curColumn));
}
// the first token is `define
// the second token is macro name
// content start from the third token
Token token = node.getFirstToken().next.next;
Token lToken = node.getLastToken();
Token prev = null;
if (token != lToken) {
while (token != null) {
Token newToken = newOneToken(token.image, curColumn);
newToken.endColumn = column;
newToken.kind = token.kind;
newToken.prev = prev;
newToken.special = true;
if (prev != null) prev.next = newToken;
// lastToken = newToken;
if (token == lToken) break;
token = token.next;
}
ret = null;
} else {
// only one token in the macro content, just return it
ret = token.image;
}
}
}
return ret;
}