public Token nextToken() {
final Token token = tokenizer.nextToken();
if (token == null) {
return token;
}
if (!token.hasTag(AreaTag.PROGRAM_TEXT_AREA)) {
return token;
}
final String text = token.getText();
if (text == null) {
return token;
}
if (PATTERN.matcher(text).matches()) {
if (LOGGER.isTraceEnabled()) {
LOGGER.trace("Marking " + token + " as a source formatting directive.");
}
token.removeTag(AreaTag.PROGRAM_TEXT_AREA);
// token.addTag(AreaTag.COMMENT);
token.addTag(AreaTag.SOURCE_FORMATTING_DIRECTIVE);
}
return token;
}
@Test
public void test() throws IOException {
final List<Token> fixedTokens = process("testsuite/koopa/test.CBL", SourceFormat.FIXED);
final List<Token> freeTokens = process("testsuite/koopa/free.CBL", SourceFormat.FREE);
boolean mismatched = false;
final int numberOfFixedTokens = fixedTokens.size();
final int numberOfFreeTokens = freeTokens.size();
final int numberOfTokens = Math.min(numberOfFixedTokens, numberOfFreeTokens);
for (int i = 0; !mismatched && i < numberOfTokens; i++) {
final Token fixedToken = fixedTokens.get(i);
final Token freeToken = freeTokens.get(i);
if (!fixedToken.getText().equals(freeToken.getText())) {
System.out.println("Mismatch starts with " + fixedToken + " vs. " + freeToken + ".");
mismatched = true;
break;
}
for (Object tag : fixedToken.getTags()) {
if (tag instanceof ContinuationsTag) {
continue;
}
if (!freeToken.hasTag(tag)) {
System.out.println(fixedToken + " has tag " + tag + ", but " + freeToken + " does not.");
mismatched = true;
}
}
for (Object tag : freeToken.getTags()) {
if (!fixedToken.hasTag(tag)) {
System.out.println(freeToken + " has tag " + tag + ", but " + fixedToken + " does not.");
mismatched = true;
}
}
}
Assert.assertFalse(
"Inputs should produce the same tokens.",
mismatched || numberOfFixedTokens != numberOfFreeTokens);
}