/*
* (non-Javadoc)
* @see com.aptana.editor.common.text.rules.DelegatingRuleBasedScanner#nextToken()
*/
@Override
public IToken nextToken() {
IToken token;
if (!hasTokens) {
hasTokens = true;
token = findBrokenToken();
if (!token.isUndefined()) {
return token;
}
}
token = super.nextToken();
if (rescanNestedLanguages
&& (doubleQuotedStringToken == token || singleQuotedStringToken == token)) {
IToken attributeToken = getAttributeToken();
ITokenScanner tokenScanner = null;
if (attributeScriptToken == attributeToken) {
tokenScanner = jsTokenScanner;
} else if (attributeStyleToken == attributeToken) {
tokenScanner = cssTokenScanner;
}
tokenHistory.clear();
int offset = getTokenOffset();
int length = getTokenLength() - 2;
if (tokenScanner != null && length > 0) {
queueToken(token, offset, 1);
queueDelegate(tokenScanner, offset + 1, length);
queueToken(token, offset + length + 1, 1);
return super.nextToken();
}
}
if (!token.isWhitespace()) {
tokenHistory.push(token);
}
return token;
}
@Override
public IToken evaluate(ICharacterScanner scanner, CombinedWordRule.CharacterBuffer word) {
IToken token = super.evaluate(scanner, word);
if (fIsVersionMatch || token.isUndefined()) {
return token;
}
return fDefaultToken;
}
@Test
public void useCase_ex1() {
when(stepLocator.findFirstStep(Mockito.anyString()))
.thenAnswer(
new Answer<StepCandidate>() {
@Override
public StepCandidate answer(InvocationOnMock invocation) throws Throwable {
System.out.println("StepScannerStyledTest.useCase_ex1(" + invocation + ")");
return candidate;
}
});
String[] expected = { //
"step_keyword ~ offset: 0, length: 6", //
"step_default ~ offset: 6, length: 18", //
"step_parameter_value ~ offset: 24, length: 6", //
"step_default ~ offset: 30, length: 33", //
"step_example_table_separator ~ offset: 63, length: 1", //
"step_example_table_cell ~ offset: 64, length: 3", //
"step_example_table_separator ~ offset: 67, length: 1", //
"step_example_table_cell ~ offset: 68, length: 5", //
"step_example_table_separator ~ offset: 73, length: 1", //
"step_default ~ offset: 74, length: 1", //
"comment ~ offset: 75, length: 17", //
"step_example_table_separator ~ offset: 92, length: 1", //
"step_example_table_cell ~ offset: 93, length: 5", //
"step_example_table_separator ~ offset: 98, length: 1", //
"step_example_table_cell ~ offset: 99, length: 6", //
"step_example_table_separator ~ offset: 105, length: 1", //
"step_default ~ offset: 106, length: 1", //
"step_example_table_separator ~ offset: 107, length: 1", //
"step_example_table_cell ~ offset: 108, length: 8", //
"step_example_table_separator ~ offset: 116, length: 1", //
"step_example_table_cell ~ offset: 117, length: 6", //
"step_example_table_separator ~ offset: 123, length: 1", //
"step_default ~ offset: 124, length: 1" //
};
int index = 0;
scanner.setRange(document, 0, document.getLength());
IToken token = scanner.nextToken();
while (!token.isEOF()) {
String actual =
(token.getData()
+ " ~ offset: "
+ scanner.getTokenOffset()
+ ", length: "
+ scanner.getTokenLength());
assertThat(actual, equalTo(expected[index++]));
token = scanner.nextToken();
}
assertThat(index, equalTo(expected.length));
}
/*
* (non-Javadoc)
*
* @see
* com.aptana.editor.common.CompositeSubPartitionScanner#setLastToken(org
* .eclipse.jface.text.rules.IToken)
*/
@Override
public void setLastToken(IToken token) {
super.setLastToken(token);
if (token == null) {
return;
}
if (!(token.getData() instanceof String)) {
current = TYPE_DEFAULT;
return;
}
String contentType = (String) token.getData();
if (HTMLSourceConfiguration.HTML_SCRIPT.equals(contentType)
|| SVGSourceConfiguration.SCRIPT.equals(contentType)) {
if (!(token instanceof ExtendedToken
&& ((HTMLUtils.isTagSelfClosing(((ExtendedToken) token).getContents()))
|| !HTMLUtils.isJavaScriptTag(((ExtendedToken) token).getContents())))) {
current = TYPE_JS;
super.setLastToken(null);
}
} else if (HTMLSourceConfiguration.HTML_STYLE.equals(contentType)
|| SVGSourceConfiguration.STYLE.equals(contentType)) {
if (!(token instanceof ExtendedToken
&& HTMLUtils.isTagSelfClosing(((ExtendedToken) token).getContents()))) {
current = TYPE_CSS;
super.setLastToken(null);
}
} else if (HTMLSourceConfiguration.HTML_SVG.equals(contentType)) {
if (!(token instanceof ExtendedToken
&& HTMLUtils.isTagSelfClosing(((ExtendedToken) token).getContents()))) {
current = TYPE_SVG;
super.setLastToken(null);
}
} else if (HTMLSourceConfiguration.DEFAULT.equals(contentType)
|| IDocument.DEFAULT_CONTENT_TYPE.equals(contentType)) {
current = TYPE_DEFAULT;
} else {
for (int i = 0; i < subPartitionScanners.length; ++i) {
if (subPartitionScanners[i].hasContentType(contentType)) {
current = i;
break;
}
}
}
}
@Override
public String toString() {
try {
return token.getData()
+ ", offset: "
+ offset
+ ", length: "
+ length
+ ", c>>"
+ document.get(offset, length)
+ "<<";
} catch (BadLocationException e) {
return token.getData()
+ ", offset: "
+ offset
+ ", length: "
+ length
+ ", c>>"
+ "//BadLocationException//"
+ "<<";
}
}
protected void emit(IToken token, int offset, int length) {
log.debug("Emitting ({}, offset: {}, length: {})", o(token.getData(), offset, length));
if (length == 0) {
log.debug(
"Empty token emitted zero length, data: {}, offset: {}, length: {}",
o(token.getData(), offset, length));
} else if (length < 0) {
log.error(
"Invalid token emitted negative length, data: {}, offset: {}, length: {}",
o(token.getData(), offset, length));
} else {
log.debug(
"Token emitted, data: {}, offset: {}, length: {}, content: <{}>",
o(token.getData(), offset, length, getContentForLog(offset, length)));
}
// can we merge previous one?
if (!fragments.isEmpty()) {
Fragment previous = getLastFragment();
// check no hole
int requiredOffset = previous.offset + previous.length;
if (offset != requiredOffset) {
log.debug(
"**hole completion**, offset: {} (vs required: {}), length: {}; previous offset: {}, length: {}",
o(offset, requiredOffset, length, previous.offset, previous.length));
emit(getDefaultToken(), requiredOffset, offset - requiredOffset);
previous = getLastFragment();
}
if (previous.token == token) {
previous.length += length;
log.debug("Token merged, offset: {}, length: {}", o(previous.offset, previous.length));
return;
}
}
Fragment fragment = new Fragment(token, offset, length);
addFragment(fragment);
}
@Override
protected boolean isIgnoredToken(IToken token) {
return super.isIgnoredToken(token) || token.getData().equals("DLTK_default"); // $NON-NLS-1$
}
private void assertColor(final IToken token, final RGB color) {
TextAttribute ta = (TextAttribute) token.getData();
assertEquals(color, ta.getForeground().getRGB());
}
@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
buffer.setLength(0);
int state = INIT_STATE;
if (resume) scanToBeginOfLine(scanner);
for (int c = scanner.read(); c != ICharacterScanner.EOF; c = scanner.read()) {
switch (state) {
case INIT_STATE:
if (c != '\n' && Character.isWhitespace((char) c)) {
break;
}
if (isValidCharacter(c)) {
state = VAR_STATE;
} else {
state = ERROR_STATE;
}
break;
case VAR_STATE:
if (isValidCharacter(c)) {
break;
}
// $FALL-THROUGH$
case END_VAR_STATE:
if (c != '\n' && Character.isWhitespace((char) c)) {
state = END_VAR_STATE;
} else if (c == ':' || c == '+' || c == '?') {
state = EQUAL_STATE;
} else if (c == '=') {
state = FINISH_STATE;
} else {
state = ERROR_STATE;
}
break;
case EQUAL_STATE:
if (c == '=') {
state = FINISH_STATE;
} else {
state = ERROR_STATE;
}
break;
case FINISH_STATE:
break;
default:
break;
}
if (state >= FINISH_STATE) {
break;
}
buffer.append((char) c);
}
scanner.unread();
if (state == FINISH_STATE) {
// scanToEndOfLine(scanner);
return token;
}
if (defaultToken.isUndefined()) unreadBuffer(scanner);
return defaultToken;
}
