コード例 #1
0
 /** Returns a concatenation of the remaining strings from a Tokenizer. */
 private String remainingStrings() throws IOException {
   StringBuffer buffer = null;
   while (true) {
     Tokenizer.Token t = get();
     if (!t.isString()) break;
     if (buffer == null) buffer = new StringBuffer();
     buffer.append(t.value);
   }
   unget();
   if (buffer == null) return null;
   return buffer.toString();
 }
コード例 #2
0
 private Token set(int type, StringBuffer value) {
   if (type < 0) throw new IllegalArgumentException();
   this.type = type;
   this.value = value == null ? null : value.toString();
   return this;
 }
コード例 #3
0
  /**
   * Gets the next token from a tokenizer.
   *
   * @param wantWhitespace If true, leading whitespace will be returned as a token.
   * @param wantComment If true, comments are returned as tokens.
   * @return The next token in the stream.
   * @throws TextParseException The input was invalid.
   * @throws IOException An I/O error occurred.
   */
  public Token get(boolean wantWhitespace, boolean wantComment) throws IOException {
    int type;
    int c;

    if (ungottenToken) {
      ungottenToken = false;
      if (current.type == WHITESPACE) {
        if (wantWhitespace) return current;
      } else if (current.type == COMMENT) {
        if (wantComment) return current;
      } else {
        if (current.type == EOL) line++;
        return current;
      }
    }
    int skipped = skipWhitespace();
    if (skipped > 0 && wantWhitespace) return current.set(WHITESPACE, null);
    type = IDENTIFIER;
    sb.setLength(0);
    while (true) {
      c = getChar();
      if (c == -1 || delimiters.indexOf(c) != -1) {
        if (c == -1) {
          if (quoting) throw exception("EOF in " + "quoted string");
          else if (sb.length() == 0) return current.set(EOF, null);
          else return current.set(type, sb);
        }
        if (sb.length() == 0 && type != QUOTED_STRING) {
          if (c == '(') {
            multiline++;
            skipWhitespace();
            continue;
          } else if (c == ')') {
            if (multiline <= 0) throw exception("invalid " + "close " + "parenthesis");
            multiline--;
            skipWhitespace();
            continue;
          } else if (c == '"') {
            if (!quoting) {
              quoting = true;
              delimiters = quotes;
              type = QUOTED_STRING;
            } else {
              quoting = false;
              delimiters = delim;
              skipWhitespace();
            }
            continue;
          } else if (c == '\n') {
            return current.set(EOL, null);
          } else if (c == ';') {
            while (true) {
              c = getChar();
              if (c == '\n' || c == -1) break;
              sb.append((char) c);
            }
            if (wantComment) {
              ungetChar(c);
              return current.set(COMMENT, sb);
            } else if (c == -1 && type != QUOTED_STRING) {
              checkUnbalancedParens();
              return current.set(EOF, null);
            } else if (multiline > 0) {
              skipWhitespace();
              sb.setLength(0);
              continue;
            } else return current.set(EOL, null);
          } else throw new IllegalStateException();
        } else ungetChar(c);
        break;
      } else if (c == '\\') {
        c = getChar();
        if (c == -1) throw exception("unterminated escape sequence");
        sb.append('\\');
      } else if (quoting && c == '\n') {
        throw exception("newline in quoted string");
      }
      sb.append((char) c);
    }
    if (sb.length() == 0 && type != QUOTED_STRING) {
      checkUnbalancedParens();
      return current.set(EOF, null);
    }
    return current.set(type, sb);
  }