public byte markTokensImpl(byte token, Segment line, int lineIndex) {
char[] array = line.array;
int offset = line.offset;
int lastOffset = offset;
int length = line.count + offset;
if (SyntaxUtilities.regionMatches(true, line, offset, "rem")) {
addToken(line.count, Token.COMMENT1);
return Token.NULL;
}
loop:
for (int i = offset; i < length; i++) {
int i1 = (i + 1);
switch (token) {
case Token.NULL:
switch (array[i]) {
case '%':
addToken(i - lastOffset, token);
lastOffset = i;
if (length - i <= 3 || array[i + 2] == ' ') {
addToken(2, Token.KEYWORD2);
i += 2;
lastOffset = i;
} else token = Token.KEYWORD2;
break;
case '"':
addToken(i - lastOffset, token);
token = Token.LITERAL1;
lastOffset = i;
break;
case ':':
if (i == offset) {
addToken(line.count, Token.LABEL);
lastOffset = length;
break loop;
}
break;
case ' ':
if (lastOffset == offset) {
addToken(i - lastOffset, Token.KEYWORD1);
lastOffset = i;
}
break;
}
break;
case Token.KEYWORD2:
if (array[i] == '%') {
addToken(i1 - lastOffset, token);
token = Token.NULL;
lastOffset = i1;
}
break;
case Token.LITERAL1:
if (array[i] == '"') {
addToken(i1 - lastOffset, token);
token = Token.NULL;
lastOffset = i1;
}
break;
default:
throw new InternalError("Invalid state: " + token);
}
}
if (lastOffset != length) {
if (token != Token.NULL) token = Token.INVALID;
else if (lastOffset == offset) token = Token.KEYWORD1;
addToken(length - lastOffset, token);
}
return Token.NULL;
}
public byte markTokensImpl(byte _token, Segment line, int lineIndex) {
char[] array = line.array;
int offset = line.offset;
token = _token;
lastOffset = offset;
lastKeyword = offset;
matchChar = '\0';
matchCharBracket = false;
matchSpacesAllowed = false;
int length = line.count + offset;
if (token == Token.LITERAL1 && lineIndex != 0 && lineInfo[lineIndex - 1].obj != null) {
String str = (String) lineInfo[lineIndex - 1].obj;
if (str != null
&& str.length() == line.count
&& SyntaxUtilities.regionMatches(false, line, offset, str)) {
addToken(line.count, token);
return Token.NULL;
} else {
addToken(line.count, token);
lineInfo[lineIndex].obj = str;
return token;
}
}
boolean backslash = false;
loop:
for (int i = offset; i < length; i++) {
int i1 = (i + 1);
char c = array[i];
if (c == '\\') {
backslash = !backslash;
continue;
}
switch (token) {
case Token.NULL:
switch (c) {
case '#':
if (doKeyword(line, i, c)) break;
if (backslash) backslash = false;
else {
addToken(i - lastOffset, token);
addToken(length - i, Token.COMMENT1);
lastOffset = lastKeyword = length;
break loop;
}
break;
case '=':
backslash = false;
if (i == offset) {
token = Token.COMMENT2;
addToken(length - i, token);
lastOffset = lastKeyword = length;
break loop;
} else doKeyword(line, i, c);
break;
case '$':
case '&':
case '%':
case '@':
backslash = false;
if (doKeyword(line, i, c)) break;
if (length - i > 1) {
if (c == '&' && (array[i1] == '&' || Character.isWhitespace(array[i1]))) i++;
else {
addToken(i - lastOffset, token);
lastOffset = lastKeyword = i;
token = Token.KEYWORD2;
}
}
break;
case '"':
if (doKeyword(line, i, c)) break;
if (backslash) backslash = false;
else {
addToken(i - lastOffset, token);
token = Token.LITERAL1;
lineInfo[lineIndex].obj = null;
lastOffset = lastKeyword = i;
}
break;
case '\'':
if (backslash) backslash = false;
else {
int oldLastKeyword = lastKeyword;
if (doKeyword(line, i, c)) break;
if (i != oldLastKeyword) break;
addToken(i - lastOffset, token);
token = Token.LITERAL2;
lastOffset = lastKeyword = i;
}
break;
case '`':
if (doKeyword(line, i, c)) break;
if (backslash) backslash = false;
else {
addToken(i - lastOffset, token);
token = Token.OPERATOR;
lastOffset = lastKeyword = i;
}
break;
case '<':
if (doKeyword(line, i, c)) break;
if (backslash) backslash = false;
else {
if (length - i > 2 && array[i1] == '<' && !Character.isWhitespace(array[i + 2])) {
addToken(i - lastOffset, token);
lastOffset = lastKeyword = i;
token = Token.LITERAL1;
int len = length - (i + 2);
if (array[length - 1] == ';') len--;
lineInfo[lineIndex].obj = createReadinString(array, i + 2, len);
}
}
break;
case ':':
backslash = false;
if (doKeyword(line, i, c)) break;
// Doesn't pick up all labels,
// but at least doesn't mess up
if (lastKeyword != 0) break;
addToken(i1 - lastOffset, Token.LABEL);
lastOffset = lastKeyword = i1;
break;
case '-':
backslash = false;
if (doKeyword(line, i, c)) break;
if (i != lastKeyword || length - i <= 1) break;
switch (array[i1]) {
case 'r':
case 'w':
case 'x':
case 'o':
case 'R':
case 'W':
case 'X':
case 'O':
case 'e':
case 'z':
case 's':
case 'f':
case 'd':
case 'l':
case 'p':
case 'S':
case 'b':
case 'c':
case 't':
case 'u':
case 'g':
case 'k':
case 'T':
case 'B':
case 'M':
case 'A':
case 'C':
addToken(i - lastOffset, token);
addToken(2, Token.KEYWORD3);
lastOffset = lastKeyword = i + 2;
i++;
}
break;
case '/':
case '?':
if (doKeyword(line, i, c)) break;
if (length - i > 1) {
backslash = false;
char ch = array[i1];
if (Character.isWhitespace(ch)) break;
matchChar = c;
matchSpacesAllowed = false;
addToken(i - lastOffset, token);
token = S_ONE;
lastOffset = lastKeyword = i;
}
break;
default:
backslash = false;
if (!Character.isLetterOrDigit(c) && c != '_') doKeyword(line, i, c);
break;
}
break;
case Token.KEYWORD2:
backslash = false;
// This test checks for an end-of-variable
// condition
if (!Character.isLetterOrDigit(c)
&& c != '_'
&& c != '#'
&& c != '\''
&& c != ':'
&& c != '&') {
// If this is the first character
// of the variable name ($'aaa)
// ignore it
if (i != offset && array[i - 1] == '$') {
addToken(i1 - lastOffset, token);
lastOffset = lastKeyword = i1;
}
// Otherwise, end of variable...
else {
addToken(i - lastOffset, token);
lastOffset = lastKeyword = i;
// Wind back so that stuff
// like $hello$fred is picked
// up
i--;
token = Token.NULL;
}
}
break;
case S_ONE:
case S_TWO:
if (backslash) backslash = false;
else {
if (matchChar == '\0') {
if (Character.isWhitespace(matchChar) && !matchSpacesAllowed) break;
else matchChar = c;
} else {
switch (matchChar) {
case '(':
matchChar = ')';
matchCharBracket = true;
break;
case '[':
matchChar = ']';
matchCharBracket = true;
break;
case '{':
matchChar = '}';
matchCharBracket = true;
break;
case '<':
matchChar = '>';
matchCharBracket = true;
break;
default:
matchCharBracket = false;
break;
}
if (c != matchChar) break;
if (token == S_TWO) {
token = S_ONE;
if (matchCharBracket) matchChar = '\0';
} else {
token = S_END;
addToken(i1 - lastOffset, Token.LITERAL2);
lastOffset = lastKeyword = i1;
}
}
}
break;
case S_END:
backslash = false;
if (!Character.isLetterOrDigit(c) && c != '_') doKeyword(line, i, c);
break;
case Token.COMMENT2:
backslash = false;
if (i == offset) {
addToken(line.count, token);
if (length - i > 3 && SyntaxUtilities.regionMatches(false, line, offset, "=cut"))
token = Token.NULL;
lastOffset = lastKeyword = length;
break loop;
}
break;
case Token.LITERAL1:
if (backslash) backslash = false;
/*
* else if(c == '$') backslash = true;
*/
else if (c == '"') {
addToken(i1 - lastOffset, token);
token = Token.NULL;
lastOffset = lastKeyword = i1;
}
break;
case Token.LITERAL2:
if (backslash) backslash = false;
/*
* else if(c == '$') backslash = true;
*/
else if (c == '\'') {
addToken(i1 - lastOffset, Token.LITERAL1);
token = Token.NULL;
lastOffset = lastKeyword = i1;
}
break;
case Token.OPERATOR:
if (backslash) backslash = false;
else if (c == '`') {
addToken(i1 - lastOffset, token);
token = Token.NULL;
lastOffset = lastKeyword = i1;
}
break;
default:
throw new InternalError("Invalid state: " + token);
}
}
if (token == Token.NULL) doKeyword(line, length, '\0');
switch (token) {
case Token.KEYWORD2:
addToken(length - lastOffset, token);
token = Token.NULL;
break;
case Token.LITERAL2:
addToken(length - lastOffset, Token.LITERAL1);
break;
case S_END:
addToken(length - lastOffset, Token.LITERAL2);
token = Token.NULL;
break;
case S_ONE:
case S_TWO:
addToken(length - lastOffset, Token.INVALID);
token = Token.NULL;
break;
default:
addToken(length - lastOffset, token);
break;
}
return token;
}
public byte markTokensImpl(byte token, Segment line, int lineIndex) {
char[] array = line.array;
byte cmdState = 0; // 0 = space before command, 1 = inside
// command, 2 = after command
int offset = line.offset;
int lastOffset = offset;
int length = line.count + offset;
if (token == Token.LITERAL1 && lineIndex != 0 && lineInfo[lineIndex - 1].obj != null) {
String str = (String) lineInfo[lineIndex - 1].obj;
if (str != null
&& str.length() == line.count
&& SyntaxUtilities.regionMatches(false, line, offset, str)) {
addToken(line.count, Token.LITERAL1);
return Token.NULL;
} else {
addToken(line.count, Token.LITERAL1);
lineInfo[lineIndex].obj = str;
return Token.LITERAL1;
}
}
boolean backslash = false;
loop:
for (int i = offset; i < length; i++) {
int i1 = (i + 1);
char c = array[i];
if (c == '\\') {
backslash = !backslash;
continue;
}
switch (token) {
case Token.NULL:
switch (c) {
case ' ':
case '\t':
case '(':
case ')':
backslash = false;
if (cmdState == 1 /* insideCmd */) {
addToken(i - lastOffset, Token.KEYWORD1);
lastOffset = i;
cmdState = 2; /* afterCmd */
}
break;
case '=':
backslash = false;
if (cmdState == 1 /* insideCmd */) {
addToken(i - lastOffset, token);
lastOffset = i;
cmdState = 2; /* afterCmd */
}
break;
case '&':
case '|':
case ';':
if (backslash) backslash = false;
else cmdState = 0; /* beforeCmd */
break;
case '#':
if (backslash) backslash = false;
else {
addToken(i - lastOffset, token);
addToken(length - i, Token.COMMENT1);
lastOffset = length;
break loop;
}
break;
case '$':
if (backslash) backslash = false;
else {
addToken(i - lastOffset, token);
cmdState = 2; /* afterCmd */
lastOffset = i;
if (length - i >= 2) {
switch (array[i1]) {
case '(':
continue;
case '{':
token = LVARIABLE;
break;
default:
token = Token.KEYWORD2;
break;
}
} else token = Token.KEYWORD2;
}
break;
case '"':
if (backslash) backslash = false;
else {
addToken(i - lastOffset, token);
token = Token.LITERAL1;
lineInfo[lineIndex].obj = null;
cmdState = 2; /* afterCmd */
lastOffset = i;
}
break;
case '\'':
if (backslash) backslash = false;
else {
addToken(i - lastOffset, token);
token = Token.LITERAL2;
cmdState = 2; /* afterCmd */
lastOffset = i;
}
break;
case '<':
if (backslash) backslash = false;
else {
if (length - i > 1 && array[i1] == '<') {
addToken(i - lastOffset, token);
token = Token.LITERAL1;
lastOffset = i;
lineInfo[lineIndex].obj = new String(array, i + 2, length - (i + 2));
}
}
break;
default:
backslash = false;
if (Character.isLetter(c)) {
if (cmdState == 0 /* beforeCmd */) {
addToken(i - lastOffset, token);
lastOffset = i;
cmdState++; /* insideCmd */
}
}
break;
}
break;
case Token.KEYWORD2:
backslash = false;
if (!Character.isLetterOrDigit(c) && c != '_') {
if (i != offset && array[i - 1] == '$') {
addToken(i1 - lastOffset, token);
lastOffset = i1;
token = Token.NULL;
continue;
} else {
addToken(i - lastOffset, token);
lastOffset = i;
token = Token.NULL;
}
}
break;
case Token.LITERAL1:
if (backslash) backslash = false;
else if (c == '"') {
addToken(i1 - lastOffset, token);
cmdState = 2; /* afterCmd */
lastOffset = i1;
token = Token.NULL;
} else backslash = false;
break;
case Token.LITERAL2:
if (backslash) backslash = false;
else if (c == '\'') {
addToken(i1 - lastOffset, Token.LITERAL1);
cmdState = 2; /* afterCmd */
lastOffset = i1;
token = Token.NULL;
} else backslash = false;
break;
case LVARIABLE:
backslash = false;
if (c == '}') {
addToken(i1 - lastOffset, Token.KEYWORD2);
lastOffset = i1;
token = Token.NULL;
}
break;
default:
throw new InternalError("Invalid state: " + token);
}
}
switch (token) {
case Token.NULL:
if (cmdState == 1) addToken(length - lastOffset, Token.KEYWORD1);
else addToken(length - lastOffset, token);
break;
case Token.LITERAL2:
addToken(length - lastOffset, Token.LITERAL1);
break;
case Token.KEYWORD2:
addToken(length - lastOffset, token);
token = Token.NULL;
break;
case LVARIABLE:
addToken(length - lastOffset, Token.INVALID);
token = Token.NULL;
break;
default:
addToken(length - lastOffset, token);
break;
}
return token;
}