@Override
 public void reset() throws IOException {
   super.reset();
   offset = 0;
   inputLength = 0;
   tokenCollector.clear();
   tokenIteractor = null;
 }
  @Override
  public boolean incrementToken() throws IOException {
    clearAttributes();

    // 已经穷尽tokensIteractor的Token对象,则继续请求reader流入数据
    while (tokenIteractor == null || !tokenIteractor.hasNext()) {
      // System.out.println(dissected);
      int read = 0;
      int remainning = -1; // 重新从reader读入字符前,buffer中还剩下的字符数,负数表示当前暂不需要从reader中读入字符
      if (dissected >= beef.length()) {
        remainning = 0;
      } else if (dissected < 0) {
        remainning = bufferLength + dissected;
      }
      if (remainning >= 0) {
        if (remainning > 0) {
          System.arraycopy(buffer, -dissected, buffer, 0, remainning);
        }
        read = input.read(buffer, remainning, bufferLength - remainning);
        inputLength += read;
        int charCount = remainning + read;
        if (charCount < 0) {
          // reader已尽,按接口next()要求返回null.
          return false;
        }
        if (charCount < bufferLength) {
          buffer[charCount++] = 0;
        }
        // 构造“牛”,并使用knife“解”之
        beef.set(0, charCount);
        offset += Math.abs(dissected);
        // offset -= remainning;
        dissected = 0;
      }
      dissected = knife.dissect(this, beef, dissected);
      // offset += read;// !!!
      tokenIteractor = tokenCollector.iterator();
    }

    if (tokenIteractor.hasNext()) {
      // 返回tokensIteractor下一个Token对象
      Token token = tokenIteractor.next();
      termAtt.setEmpty();
      termAtt.append(token.charSequence());
      offsetAtt.setOffset(correctOffset(token.startOffset()), correctOffset(token.endOffset()));
      positionIncrementAttribute.setPositionIncrement(token.endOffset());
      return true;
    }
    return tokenIteractor.hasNext();
  }
 @Override
 public Token next() throws IOException {
   //
   while (tokenIteractor == null || !tokenIteractor.hasNext()) {
     int read = 0;
     int remainning = -1; // the remaining word in buffer."-1" means that
     // no word in buffer
     if (dissected >= beef.length()) {
       remainning = 0;
     } else if (dissected < 0) {
       remainning = bufferLength + dissected;
     }
     if (remainning != -1) {
       if (remainning > 0) {
         System.arraycopy(buffer, -dissected, buffer, 0, remainning);
       }
       read = input.read(buffer, remainning, bufferLength - remainning);
       int charCount = remainning + read;
       if (charCount < 0) {
         // the reader in complete. return
         return null;
       }
       if (charCount < bufferLength) {
         buffer[charCount++] = 0;
       }
       // construct the dictionary. use knife to analyzer
       beef.set(0, charCount);
       offset -= remainning;
       dissected = 0;
     }
     dissected = knife.dissect((Collector) this, beef, dissected);
     offset += read; // !!!
     tokenIteractor = tokenCollector.iterator();
   }
   // return tokensIteractor's next Token
   return tokenIteractor.next();
 }
 public void collect(String word, int offset, int end) {
   tokenCollector.collect(word, this.offset + offset, this.offset + end);
 }
 private static void doTest(final String text, final String... expected) {
   TokenCollector collector = new TokenCollector();
   LiteralExpressionTokenizer.processTextWithEscapeSequences(null, text, collector);
   assertOrderedEquals(collector.getTokenTexts(), expected);
 }