Example #1
0
  /**
   * Creates a linked list of TokenSeps from the comma separated string str.
   *
   * @param str The string specifying delimiter strings.
   * @return A list of TokenSeps.
   */
  private static TokenSep parseDelimString(String str) {
    TokenSep first = null;
    TokenSep p = null;
    int idx0, idx1, length;
    StringBuilder val = new StringBuilder();
    char c;

    length = str.length();
    for (idx0 = 0; idx0 < length; ) {
      for (idx1 = idx0; idx1 < length; idx1++) {
        c = str.charAt(idx1);
        if (c == '\\') {
          idx1++;
          if (idx1 < length) val.append(str.charAt(idx1));
        } else if (c == ',') {
          break;
        } else {
          val.append(c);
        }
      }
      idx1 = Math.min(idx1, length);
      if (idx1 > idx0) {
        p = new TokenSep(val.toString());
        val = new StringBuilder();
        p.setNext(first);
        first = p;
      }

      idx0 = idx1 + 1;
    }

    return first;
  }
Example #2
0
  /**
   * Retrives the next token.
   *
   * @return The next token.
   */
  public String nextToken() {
    CustomSeparator csep;
    TokenSep sep;
    String s = null;
    int i, j;

    if (putToken != null) {
      s = putToken;
      putToken = null;
      return s;
    }

    if (savedToken != null) {
      s = savedToken;
      tokIdx = savedIdx;
      savedToken = null;
      return s;
    }

    if (sIdx >= eIdx) throw new NoSuchElementException("No more tokens available");

    for (sep = delims; sep != null; sep = sep.getNext()) sep.reset();

    if (customSeps != null) {
      for (i = 0; i < customSeps.size(); i++) ((CustomSeparator) customSeps.get(i)).reset();
    }

    for (i = sIdx; i < eIdx; i++) {
      char c = source.charAt(i);

      for (j = 0; customSeps != null && j < customSeps.size(); j++) {
        csep = (CustomSeparator) customSeps.get(j);

        if (csep.addChar(c)) break;
      }
      if (customSeps != null && j < customSeps.size()) {
        csep = (CustomSeparator) customSeps.get(j);

        while (csep.hasFreePart() && i + 1 < eIdx) if (csep.endChar(source.charAt(++i))) break;
        i -= Math.min(csep.getPeekCount(), i);

        int clen = Math.min(i + 1, source.length());

        if (i - sIdx + 1 > csep.tokenLength()) {
          s = source.substring(sIdx, i - csep.tokenLength() + 1);

          savedIdx = i - csep.tokenLength() + 1;
          savedToken = source.substring(savedIdx, clen);
        } else {
          s = source.substring(sIdx, clen);
        }

        tokIdx = sIdx;
        sIdx = i + 1;
        break;
      }

      for (sep = delims; sep != null; sep = sep.getNext()) if (sep.addChar(c)) break;
      if (sep != null) {
        if (i - sIdx + 1 > sep.length()) {
          s = source.substring(sIdx, i - sep.length() + 1);
          savedIdx = i - sep.length() + 1;
          savedToken = sep.getString();
        } else {
          s = sep.getString();
        }
        tokIdx = sIdx;
        sIdx = i + 1;
        break;
      }
    }

    if (s == null) {
      s = source.substring(sIdx);
      tokIdx = sIdx;
      sIdx = eIdx;
    }

    return s;
  }