Esempio n. 1
0
  /**
   * Squashes a list of tokens into one.
   *
   * @param tokenType is the type of the new token.
   * @param tokens is the list of tokens to be concatenated.
   * @return the new token.
   */
  public static Token concat(final TokenType tokenType, final List<Token> tokens) {
    final Token first = tokens.get(0);

    for (int i = 1; i < tokens.size(); i++) {
      first.addAfter(tokenType, tokens.get(i));
    }

    return first;
  }
Esempio n. 2
0
  @Override
  public Token produce(final T reader, final P productionStack, final B builder) {
    for (; ; ) {
      final Token token = reader.peek();

      if (token == null || token == EndOfStreamToken.EOS) {
        return EndOfStreamToken.EOS;
      } else if (token.type().isWhitespace()) {
        reader.next(); // Discard Whitespace
      } else {
        this.p.produce(reader, productionStack, builder);
        return null;
      }
    }
  }