Exemplo n.º 1
1
  public char[] toCharArray() {
    int l = currentString.length();
    char[] buffer = new char[l];
    currentString.getChars(0, l, buffer, 0);

    return buffer;
  }
 /** @return Return the buffer's current value as a string. */
 @Nonnull
 @ReturnsMutableCopy
 public char[] getAsCharArray() {
   final int nChars = m_aSB.length();
   final char[] ret = new char[nChars];
   m_aSB.getChars(0, nChars, ret, 0);
   return ret;
 }
Exemplo n.º 3
1
  /**
   * Tag changes in the given <var>line1</var> and <var>line2</var> for highlighting. Removed parts
   * are tagged with CSS class {@code d}, new parts are tagged with CSS class {@code a} using a
   * {@code span} element.
   *
   * @param line1 line of the original file
   * @param line2 line of the changed/new file
   * @return the tagged lines (field[0] ~= line1, field[1] ~= line2).
   * @throws NullPointerException if one of the given parameters is {@code null}.
   */
  public static String[] diffline(StringBuilder line1, StringBuilder line2) {
    int m = line1.length();
    int n = line2.length();
    if (n == 0 || m == 0) {
      return new String[] {line1.toString(), line2.toString()};
    }

    int s = 0;
    char[] csl1 = new char[m + SPAN_LEN];
    line1.getChars(0, m--, csl1, 0);
    char[] csl2 = new char[n + SPAN_LEN];
    line2.getChars(0, n--, csl2, 0);
    while (s <= m && s <= n && csl1[s] == csl2[s]) {
      s++;
    }
    while (s <= m && s <= n && csl1[m] == csl2[n]) {
      m--;
      n--;
    }

    String[] ret = new String[2];
    // deleted
    if (s <= m) {
      m++;
      System.arraycopy(csl1, m, csl1, m + SPAN_LEN, line1.length() - m);
      System.arraycopy(csl1, s, csl1, s + SPAN_D.length(), m - s);
      SPAN_E.getChars(0, SPAN_E.length(), csl1, m + SPAN_D.length());
      SPAN_D.getChars(0, SPAN_D.length(), csl1, s);
      ret[0] = new String(csl1);
    } else {
      ret[0] = line1.toString();
    }
    // added
    if (s <= n) {
      n++;
      System.arraycopy(csl2, n, csl2, n + SPAN_LEN, line2.length() - n);
      System.arraycopy(csl2, s, csl2, s + SPAN_A.length(), n - s);
      SPAN_E.getChars(0, SPAN_E.length(), csl2, n + SPAN_A.length());
      SPAN_A.getChars(0, SPAN_A.length(), csl2, s);
      ret[1] = new String(csl2);
    } else {
      ret[1] = line2.toString();
    }
    return ret;
  }
    @Override
    public void flush() {
      char tmp[] = new char[str.length()];
      str.getChars(0, str.length() - 1, tmp, 0);
      String s = (new String(tmp));

      FormattedConsole console = FormattedConsole.getDefault();
      console.print(_level, s);

      str = new StringBuilder();
    }
Exemplo n.º 5
1
 private void flushLineBuffer() throws IOException {
   int rest = lineBuffer.length();
   int cursor = 0;
   while (rest > 0) {
     int chunkSize = Math.min(rest, writeBuffer.length);
     lineBuffer.getChars(cursor, cursor + chunkSize, writeBuffer, 0);
     writer.write(writeBuffer, 0, chunkSize);
     rest -= chunkSize;
     cursor += chunkSize;
   }
   lineBuffer.setLength(0);
 }
  @Override
  public final boolean incrementToken() throws IOException {
    clearAttributes();
    if (delimitersCount == -1) {
      int length = 0;
      delimiterPositions.add(0);
      while (true) {
        int c = input.read();
        if (c < 0) {
          break;
        }
        length++;
        if (c == delimiter) {
          delimiterPositions.add(length);
          resultToken.append(replacement);
        } else {
          resultToken.append((char) c);
        }
      }
      delimitersCount = delimiterPositions.size();
      if (delimiterPositions.get(delimitersCount - 1) < length) {
        delimiterPositions.add(length);
        delimitersCount++;
      }
      if (resultTokenBuffer.length < resultToken.length()) {
        resultTokenBuffer = new char[resultToken.length()];
      }
      resultToken.getChars(0, resultToken.length(), resultTokenBuffer, 0);
      resultToken.setLength(0);
      int idx = delimitersCount - 1 - skip;
      if (idx >= 0) {
        // otherwise its ok, because we will skip and return false
        endPosition = delimiterPositions.get(idx);
      }
      finalOffset = correctOffset(length);
      posAtt.setPositionIncrement(1);
    } else {
      posAtt.setPositionIncrement(0);
    }

    while (skipped < delimitersCount - skip - 1) {
      int start = delimiterPositions.get(skipped);
      termAtt.copyBuffer(resultTokenBuffer, start, endPosition - start);
      offsetAtt.setOffset(correctOffset(start), correctOffset(endPosition));
      skipped++;
      return true;
    }

    return false;
  }
Exemplo n.º 7
0
  private void flushAttributes() throws IOException {

    // a little complex, but the end result is, potentially, two
    // fewer temp objects created per call.
    StringBuilder b = attributesBuffer.getBuffer();
    int totalLength = b.length();
    if (totalLength != 0) {
      int curIdx = 0;
      while (curIdx < totalLength) {
        if ((totalLength - curIdx) > buffer.length) {
          int end = curIdx + buffer.length;
          b.getChars(curIdx, end, buffer, 0);
          writer.write(buffer);
          curIdx += buffer.length;
        } else {
          int len = totalLength - curIdx;
          b.getChars(curIdx, curIdx + len, buffer, 0);
          writer.write(buffer, 0, len);
          curIdx += len;
        }
      }
      attributesBuffer.reset();
    }
  }
  /** Writes the joined unhyphenated term */
  private void unhyphenate() {
    int endOffset = offsetAttribute.endOffset();

    restoreState(savedState);
    savedState = null;

    char term[] = termAttribute.buffer();
    int length = hyphenated.length();
    if (length > termAttribute.length()) {
      term = termAttribute.resizeBuffer(length);
    }

    hyphenated.getChars(0, length, term, 0);
    termAttribute.setLength(length);
    offsetAttribute.setOffset(offsetAttribute.startOffset(), endOffset);
    hyphenated.setLength(0);
  }
Exemplo n.º 9
0
    private char[] toHex(char[] password) {
      StringBuilder sb = new StringBuilder();
      for (char c : password) {
        // toHexString does not prepend 0 so we have to
        if (((byte) c > -1) && (byte) c < 0x10) {
          sb.append(0);
        }

        sb.append(Integer.toHexString(c & 0xFF));
      }

      // Extract the hex string as char[]
      char[] hex = new char[sb.length()];

      sb.getChars(0, sb.length(), hex, 0);

      return hex;
    }
Exemplo n.º 10
0
    /** Writes the concatenation to the attributes */
    void write() {
      clearAttributes();
      if (termAttribute.length() < buffer.length()) {
        termAttribute.resizeBuffer(buffer.length());
      }
      char termbuffer[] = termAttribute.buffer();

      buffer.getChars(0, buffer.length(), termbuffer, 0);
      termAttribute.setLength(buffer.length());

      if (hasIllegalOffsets) {
        offsetAttribute.setOffset(savedStartOffset, savedEndOffset);
      } else {
        offsetAttribute.setOffset(startOffset, endOffset);
      }
      posIncAttribute.setPositionIncrement(position(true));
      typeAttribute.setType(savedType);
      accumPosInc = 0;
    }
Exemplo n.º 11
0
    @Override
    public void bindView(View view, Context context, Cursor cursor) {
      ViewHolder vh = (ViewHolder) view.getTag();

      cursor.copyStringToBuffer(mTitleIdx, vh.buffer1);
      vh.line1.setText(vh.buffer1.data, 0, vh.buffer1.sizeCopied);

      final int secondUnit = 1000;
      int secs = cursor.getInt(mDurationIdx) / secondUnit;
      if (secs == 0) {
        vh.duration.setText("");
      } else {
        vh.duration.setText(MusicUtils.makeTimeString(context, secs));
      }

      final StringBuilder builder = mBuilder;
      builder.delete(0, builder.length());

      String name = cursor.getString(mAlbumIdx);
      if (name == null || name.equals("<unknown>")) {
        builder.append(mUnknownAlbum);
      } else {
        builder.append(name);
      }
      builder.append('\n');
      name = cursor.getString(mArtistIdx);
      if (name == null || name.equals("<unknown>")) {
        builder.append(mUnknownArtist);
      } else {
        builder.append(name);
      }
      int len = builder.length();
      if (vh.buffer2.length < len) {
        vh.buffer2 = new char[len];
      }
      builder.getChars(0, len, vh.buffer2, 0);
      vh.line2.setText(vh.buffer2, 0, len);

      // Update the checkbox of the item, based on which the user last
      // selected.  Note that doing it this way means we must have the
      // list view update all of its items when the selected item
      // changes.
      final long id = cursor.getLong(mIdIdx);
      vh.radio.setChecked(id == mSelectedId);
      MusicLogUtils.v(
          TAG,
          "Binding id="
              + id
              + " sel="
              + mSelectedId
              + " playing="
              + mPlayingId
              + " cursor="
              + cursor);

      // Likewise, display the "now playing" icon if this item is
      // currently being previewed for the user.
      ImageView iv = vh.play_indicator;
      if (id == mPlayingId) {
        iv.setVisibility(View.VISIBLE);
      } else {
        /// M: if current song is not playing , we don't need the ImageView of play_indicator
        // visible.
        iv.setVisibility(View.GONE);
      }

      /// M:  Show drm lock when necessary @{
      updateDrmLockIcon(vh.drmLock, cursor, id);
    }
Exemplo n.º 12
0
    @Override
    public void bindView(View view, Context context, Cursor cursor) {
      ViewHolder vh = (ViewHolder) view.getTag();

      cursor.copyStringToBuffer(mTitleIdx, vh.buffer1);
      vh.line1.setText(vh.buffer1.data, 0, vh.buffer1.sizeCopied);

      int secs = cursor.getInt(mDurationIdx) / 1000;
      if (secs == 0) {
        vh.duration.setText("");
      } else {
        vh.duration.setText(MusicUtils.makeTimeString(context, secs));
      }

      final StringBuilder builder = mBuilder;
      builder.delete(0, builder.length());

      String name = cursor.getString(mAlbumIdx);
      if (name == null || name.equals("<unknown>")) {
        builder.append(mUnknownAlbum);
      } else {
        builder.append(name);
      }
      builder.append('\n');
      name = cursor.getString(mArtistIdx);
      if (name == null || name.equals("<unknown>")) {
        builder.append(mUnknownArtist);
      } else {
        builder.append(name);
      }
      int len = builder.length();
      if (vh.buffer2.length < len) {
        vh.buffer2 = new char[len];
      }
      builder.getChars(0, len, vh.buffer2, 0);
      vh.line2.setText(vh.buffer2, 0, len);

      // Update the checkbox of the item, based on which the user last
      // selected.  Note that doing it this way means we must have the
      // list view update all of its items when the selected item
      // changes.
      final long id = cursor.getLong(mIdIdx);
      vh.radio.setChecked(id == mSelectedId);
      MusicLogUtils.v(
          TAG,
          "Binding id="
              + id
              + " sel="
              + mSelectedId
              + " playing="
              + mPlayingId
              + " cursor="
              + cursor);

      // Likewise, display the "now playing" icon if this item is
      // currently being previewed for the user.
      ImageView iv = vh.play_indicator;
      if (id == mPlayingId) {
        iv.setImageResource(R.drawable.indicator_ic_mp_playing_list);
        iv.setVisibility(View.VISIBLE);
      } else {
        iv.setVisibility(View.GONE);
      }

      // Show drm lock when necessary
      iv = vh.drm_lock;
      if (FeatureOption.MTK_DRM_APP) {
        int isDRM = cursor.getInt(mIsDrmIdx);
        int drmMethod = cursor.getInt(mDrmMethodIdx);
        MusicLogUtils.d(TAG, "bindView(" + view + "): isDRM=" + isDRM + ", drmMethod=" + drmMethod);
        try {
          if (isDRM == 1 && drmMethod != DrmStore.DrmMethod.METHOD_FL) {
            if (mDrmClient.checkRightsStatus(
                    ContentUris.withAppendedId(MediaStore.Audio.Media.EXTERNAL_CONTENT_URI, id),
                    DrmStore.Action.PLAY)
                == DrmStore.RightsStatus.RIGHTS_VALID) {
              iv.setImageResource(com.mediatek.internal.R.drawable.drm_green_lock);
            } else {
              iv.setImageResource(com.mediatek.internal.R.drawable.drm_red_lock);
            }
            iv.setVisibility(View.VISIBLE);
          } else {
            iv.setVisibility(View.GONE);
          }
        } catch (Exception ex) {
          MusicLogUtils.e(TAG, "bindView: ", ex);
          iv.setVisibility(View.GONE);
        }
      } else {
        iv.setVisibility(View.GONE);
      }
    }
Exemplo n.º 13
0
  private void replaceArgs(
      PreprocessorMacro macro, TokenList[] args, TokenList[] expandedArgs, TokenList result) {
    TokenList replacement = clone(macro.getTokens(fDefinitionParser, fLexOptions, this));

    Token l = null;
    Token n;
    Token pasteArg1 = null;
    for (Token t = replacement.first(); t != null; l = t, t = n) {
      n = (Token) t.getNext();

      switch (t.getType()) {
        case ObjCPreprocessor.tMACRO_PARAMETER:
          int idx = ((TokenParameterReference) t).getIndex();
          if (idx < args.length) { // be defensive
            addSpacemarker(l, t, result); // start argument
            // replacement
            if (isKind(n, IToken.tPOUNDPOUND)) {
              TokenList arg = clone(args[idx]);
              pasteArg1 = arg.last();
              if (pasteArg1 != null) {
                result.appendAllButLast(arg);
                addSpacemarker(result.last(), pasteArg1, result); // start
                // token
                // paste
              }
            } else {
              TokenList arg = clone(expandedArgs[idx]);
              result.appendAll(arg);
              addSpacemarker(t, n, result); // end argument
              // replacement
            }
          }
          break;

        case IToken.tPOUND:
          addSpacemarker(l, t, result); // start stringify
          StringBuilder buf = new StringBuilder();
          buf.append('"');
          if (isKind(n, ObjCPreprocessor.tMACRO_PARAMETER)) {
            idx = ((TokenParameterReference) n).getIndex();
            if (idx < args.length) { // be defensive
              stringify(args[idx], buf);
            }
            t = n;
            n = (Token) n.getNext();
          }
          buf.append('"');
          final int length = buf.length();
          final char[] image = new char[length];
          buf.getChars(0, length, image, 0);

          Token generated = new TokenWithImage(IToken.tSTRING, null, 0, 0, image);
          if (isKind(n, IToken.tPOUNDPOUND)) { // start token paste,
            // same as start
            // stringify
            pasteArg1 = generated;
          } else {
            result.append(generated);
            addSpacemarker(t, n, result); // end stringify
          }
          break;

        case IToken.tPOUNDPOUND:
          Token pasteArg2 = null;
          TokenList rest = null;
          if (n != null) {
            if (n.getType() == ObjCPreprocessor.tMACRO_PARAMETER) {
              TokenList arg;
              idx = ((TokenParameterReference) n).getIndex();
              if (idx < args.length) { // be defensive
                arg = clone(args[idx]);
                pasteArg2 = arg.first();
                if (pasteArg2 != null && arg.first() != arg.last()) {
                  rest = arg;
                  rest.removeFirst();
                }
              }
            } else {
              idx = -1;
              pasteArg2 = n;
            }

            t = n;
            n = (Token) n.getNext();
            final boolean pasteNext = isKind(n, IToken.tPOUNDPOUND);

            generated = tokenpaste(pasteArg1, pasteArg2, macro);
            pasteArg1 = null;

            if (generated != null) {
              if (pasteNext && rest == null) {
                pasteArg1 = generated; // no need to mark
                // spaces, done ahead
              } else {
                result.append(generated);
                addSpacemarker(pasteArg2, rest == null ? n : rest.first(), result); // end
                // token
                // paste
              }
            }
            if (rest != null) {
              if (pasteNext) {
                pasteArg1 = rest.last();
                if (pasteArg1 != null) {
                  result.appendAllButLast(rest);
                  addSpacemarker(result.last(), pasteArg1, result); // start
                  // token
                  // paste
                }
              } else {
                result.appendAll(rest);
                if (idx >= 0) {
                  addSpacemarker(t, n, result); // end
                  // argument
                  // replacement
                }
              }
            }
          }
          break;

        case IToken.tCOMMA:
          if (isKind(n, IToken.tPOUNDPOUND)) {
            final Token nn = (Token) n.getNext();
            if (isKind(nn, ObjCPreprocessor.tMACRO_PARAMETER)) {
              idx = ((TokenParameterReference) nn).getIndex();

              // check for gcc-extension preventing the paste
              // operation
              if (idx == args.length - 1
                  && macro.hasVarArgs() != FunctionStyleMacro.NO_VAARGS
                  && !isKind(nn.getNext(), IToken.tPOUNDPOUND)) {
                final Token nnn = (Token) nn.getNext();
                TokenList arg = clone(expandedArgs[idx]);
                if (arg.isEmpty()) {
                  addSpacemarker(l, t, result);
                  addSpacemarker(nn, nnn, result);
                } else {
                  result.append(t);
                  addSpacemarker(t, n, result);
                  result.appendAll(arg);
                  addSpacemarker(nn, nnn, result);
                }
                t = nn;
                n = nnn;
                break;
              }
            }

            addSpacemarker(l, t, result);
            pasteArg1 = t;
          } else {
            result.append(t);
          }
          break;

        default:
          if (isKind(n, IToken.tPOUNDPOUND)) {
            addSpacemarker(l, t, result); // start token paste
            pasteArg1 = t;
          } else {
            result.append(t);
          }
          break;
      }
    }
  }
Exemplo n.º 14
0
  /**
   * Write directly from our FastStringWriter to the provided writer.
   *
   * @throws IOException if an error occurs
   */
  public void flushToWriter() throws IOException {

    // Save the state to a new instance of StringWriter to
    // avoid multiple serialization steps if the view contains
    // multiple forms.
    StateManager stateManager = Util.getStateManager(context);
    ResponseWriter origWriter = context.getResponseWriter();
    StringBuilder stateBuilder = getState(stateManager, origWriter);
    StringBuilder builder = fWriter.getBuffer();
    // begin writing...
    int totalLen = builder.length();
    int stateLen = stateBuilder.length();
    int pos = 0;
    int tildeIdx = getNextDelimiterIndex(builder, pos);
    while (pos < totalLen) {
      if (tildeIdx != -1) {
        if (tildeIdx > pos && (tildeIdx - pos) > bufSize) {
          // there's enough content before the first ~
          // to fill the entire buffer
          builder.getChars(pos, (pos + bufSize), buf, 0);
          orig.write(buf);
          pos += bufSize;
        } else {
          // write all content up to the first '~'
          builder.getChars(pos, tildeIdx, buf, 0);
          int len = (tildeIdx - pos);
          orig.write(buf, 0, len);
          // now check to see if the state saving string is
          // at the begining of pos, if so, write our
          // state out.
          if (builder.indexOf(RIConstants.SAVESTATE_FIELD_MARKER, pos) == tildeIdx) {
            // buf is effectively zero'd out at this point
            int statePos = 0;
            while (statePos < stateLen) {
              if ((stateLen - statePos) > bufSize) {
                // enough state to fill the buffer
                stateBuilder.getChars(statePos, (statePos + bufSize), buf, 0);
                orig.write(buf);
                statePos += bufSize;
              } else {
                int slen = (stateLen - statePos);
                stateBuilder.getChars(statePos, stateLen, buf, 0);
                orig.write(buf, 0, slen);
                statePos += slen;
              }
            }
            // push us past the last '~' at the end of the marker
            pos += (len + STATE_MARKER_LEN);
            tildeIdx = getNextDelimiterIndex(builder, pos);

            stateBuilder = getState(stateManager, origWriter);
            stateLen = stateBuilder.length();
          } else {
            pos = tildeIdx;
            tildeIdx = getNextDelimiterIndex(builder, tildeIdx + 1);
          }
        }
      } else {
        // we've written all of the state field markers.
        // finish writing content
        if (totalLen - pos > bufSize) {
          // there's enough content to fill the buffer
          builder.getChars(pos, (pos + bufSize), buf, 0);
          orig.write(buf);
          pos += bufSize;
        } else {
          // we're near the end of the response
          builder.getChars(pos, totalLen, buf, 0);
          int len = (totalLen - pos);
          orig.write(buf, 0, len);
          pos += (len + 1);
        }
      }
    }

    // all state has been written.  Have 'out' point to the
    // response so that all subsequent writes will make it to the
    // browser.
    out = orig;
  }
Exemplo n.º 15
0
  /**
   * Create the {@link SegGraph} for a sentence.
   *
   * @param sentence input sentence, without start and end markers
   * @return {@link SegGraph} corresponding to the input sentence.
   */
  @SuppressWarnings("fallthrough")
  private SegGraph createSegGraph(String sentence) {
    int i = 0, j;
    int length = sentence.length();
    int foundIndex;
    int[] charTypeArray = getCharTypes(sentence);
    StringBuilder wordBuf = new StringBuilder();
    SegToken token;
    int frequency = 0; // the number of times word appears.
    boolean hasFullWidth;
    int wordType;
    char[] charArray;

    SegGraph segGraph = new SegGraph();
    while (i < length) {
      hasFullWidth = false;
      switch (charTypeArray[i]) {
        case CharType.SPACE_LIKE:
          i++;
          break;
        case CharType.HANZI:
          j = i + 1;
          wordBuf.delete(0, wordBuf.length());
          // It doesn't matter if a single Chinese character (Hanzi) can form a phrase or not,
          // it will store that single Chinese character (Hanzi) in the SegGraph.  Otherwise, it
          // will
          // cause word division.
          wordBuf.append(sentence.charAt(i));
          charArray = new char[] {sentence.charAt(i)};
          frequency = wordDict.getFrequency(charArray);
          token = new SegToken(charArray, i, j, WordType.CHINESE_WORD, frequency);
          segGraph.addToken(token);

          foundIndex = wordDict.getPrefixMatch(charArray);
          while (j <= length && foundIndex != -1) {
            if (wordDict.isEqual(charArray, foundIndex) && charArray.length > 1) {
              // It is the phrase we are looking for; In other words, we have found a phrase
              // SegToken
              // from i to j.  It is not a monosyllabic word (single word).
              frequency = wordDict.getFrequency(charArray);
              token = new SegToken(charArray, i, j, WordType.CHINESE_WORD, frequency);
              segGraph.addToken(token);
            }

            while (j < length && charTypeArray[j] == CharType.SPACE_LIKE) j++;

            if (j < length && charTypeArray[j] == CharType.HANZI) {
              wordBuf.append(sentence.charAt(j));
              charArray = new char[wordBuf.length()];
              wordBuf.getChars(0, charArray.length, charArray, 0);
              // idArray has been found (foundWordIndex!=-1) as a prefix before.
              // Therefore, idArray after it has been lengthened can only appear after
              // foundWordIndex.
              // So start searching after foundWordIndex.
              foundIndex = wordDict.getPrefixMatch(charArray, foundIndex);
              j++;
            } else {
              break;
            }
          }
          i++;
          break;
        case CharType.FULLWIDTH_LETTER:
          hasFullWidth = true; /* intentional fallthrough */
        case CharType.LETTER:
          j = i + 1;
          while (j < length
              && (charTypeArray[j] == CharType.LETTER
                  || charTypeArray[j] == CharType.FULLWIDTH_LETTER)) {
            if (charTypeArray[j] == CharType.FULLWIDTH_LETTER) hasFullWidth = true;
            j++;
          }
          // Found a Token from i to j. Type is LETTER char string.
          charArray = Utility.STRING_CHAR_ARRAY;
          frequency = wordDict.getFrequency(charArray);
          wordType = hasFullWidth ? WordType.FULLWIDTH_STRING : WordType.STRING;
          token = new SegToken(charArray, i, j, wordType, frequency);
          segGraph.addToken(token);
          i = j;
          break;
        case CharType.FULLWIDTH_DIGIT:
          hasFullWidth = true; /* intentional fallthrough */
        case CharType.DIGIT:
          j = i + 1;
          while (j < length
              && (charTypeArray[j] == CharType.DIGIT
                  || charTypeArray[j] == CharType.FULLWIDTH_DIGIT)) {
            if (charTypeArray[j] == CharType.FULLWIDTH_DIGIT) hasFullWidth = true;
            j++;
          }
          // Found a Token from i to j. Type is NUMBER char string.
          charArray = Utility.NUMBER_CHAR_ARRAY;
          frequency = wordDict.getFrequency(charArray);
          wordType = hasFullWidth ? WordType.FULLWIDTH_NUMBER : WordType.NUMBER;
          token = new SegToken(charArray, i, j, wordType, frequency);
          segGraph.addToken(token);
          i = j;
          break;
        case CharType.DELIMITER:
          j = i + 1;
          // No need to search the weight for the punctuation.  Picking the highest frequency will
          // work.
          frequency = Utility.MAX_FREQUENCE;
          charArray = new char[] {sentence.charAt(i)};
          token = new SegToken(charArray, i, j, WordType.DELIMITER, frequency);
          segGraph.addToken(token);
          i = j;
          break;
        default:
          j = i + 1;
          // Treat the unrecognized char symbol as unknown string.
          // For example, any symbol not in GB2312 is treated as one of these.
          charArray = Utility.STRING_CHAR_ARRAY;
          frequency = wordDict.getFrequency(charArray);
          token = new SegToken(charArray, i, j, WordType.STRING, frequency);
          segGraph.addToken(token);
          i = j;
          break;
      }
    }

    // Add two more Tokens: "beginning xx beginning"
    charArray = Utility.START_CHAR_ARRAY;
    frequency = wordDict.getFrequency(charArray);
    token = new SegToken(charArray, -1, 0, WordType.SENTENCE_BEGIN, frequency);
    segGraph.addToken(token);

    // "end xx end"
    charArray = Utility.END_CHAR_ARRAY;
    frequency = wordDict.getFrequency(charArray);
    token = new SegToken(charArray, length, length + 1, WordType.SENTENCE_END, frequency);
    segGraph.addToken(token);

    return segGraph;
  }