public Token nextToken() {
    final Token token = tokenizer.nextToken();

    if (token == null) {
      return token;
    }

    if (!token.hasTag(AreaTag.PROGRAM_TEXT_AREA)) {
      return token;
    }

    final String text = token.getText();

    if (text == null) {
      return token;
    }

    if (PATTERN.matcher(text).matches()) {
      if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Marking " + token + " as a source formatting directive.");
      }

      token.removeTag(AreaTag.PROGRAM_TEXT_AREA);
      // token.addTag(AreaTag.COMMENT);
      token.addTag(AreaTag.SOURCE_FORMATTING_DIRECTIVE);
    }

    return token;
  }
예제 #2
0
  @Test
  public void test() throws IOException {
    final List<Token> fixedTokens = process("testsuite/koopa/test.CBL", SourceFormat.FIXED);
    final List<Token> freeTokens = process("testsuite/koopa/free.CBL", SourceFormat.FREE);

    boolean mismatched = false;
    final int numberOfFixedTokens = fixedTokens.size();
    final int numberOfFreeTokens = freeTokens.size();
    final int numberOfTokens = Math.min(numberOfFixedTokens, numberOfFreeTokens);

    for (int i = 0; !mismatched && i < numberOfTokens; i++) {
      final Token fixedToken = fixedTokens.get(i);
      final Token freeToken = freeTokens.get(i);

      if (!fixedToken.getText().equals(freeToken.getText())) {
        System.out.println("Mismatch starts with " + fixedToken + " vs. " + freeToken + ".");

        mismatched = true;
        break;
      }

      for (Object tag : fixedToken.getTags()) {
        if (tag instanceof ContinuationsTag) {
          continue;
        }

        if (!freeToken.hasTag(tag)) {
          System.out.println(fixedToken + " has tag " + tag + ", but " + freeToken + " does not.");
          mismatched = true;
        }
      }

      for (Object tag : freeToken.getTags()) {
        if (!fixedToken.hasTag(tag)) {
          System.out.println(freeToken + " has tag " + tag + ", but " + fixedToken + " does not.");
          mismatched = true;
        }
      }
    }

    Assert.assertFalse(
        "Inputs should produce the same tokens.",
        mismatched || numberOfFixedTokens != numberOfFreeTokens);
  }