/** {@inheritDoc} */
  @Override
  public boolean incrementToken() throws IOException {
    while (input.incrementToken()) {
      char[] term = termAttribute.buffer();
      int termLength = termAttribute.length();

      if (termLength > 0 && term[termLength - 1] == '-') {
        // a hyphenated word
        // capture the state of the first token only
        if (savedState == null) {
          savedState = captureState();
        }
        hyphenated.append(term, 0, termLength - 1);
      } else if (savedState == null) {
        // not part of a hyphenated word.
        return true;
      } else {
        // the final portion of a hyphenated word
        hyphenated.append(term, 0, termLength);
        unhyphenate();
        return true;
      }
    }

    if (savedState != null) {
      // the final term ends with a hyphen
      // add back the hyphen, for backwards compatibility.
      hyphenated.append('-');
      unhyphenate();
      return true;
    }

    return false;
  }
  /** Writes the joined unhyphenated term */
  private void unhyphenate() {
    int endOffset = offsetAttribute.endOffset();

    restoreState(savedState);
    savedState = null;

    char term[] = termAttribute.buffer();
    int length = hyphenated.length();
    if (length > termAttribute.length()) {
      term = termAttribute.resizeBuffer(length);
    }

    hyphenated.getChars(0, length, term, 0);
    termAttribute.setLength(length);
    offsetAttribute.setOffset(offsetAttribute.startOffset(), endOffset);
    hyphenated.setLength(0);
  }
Ejemplo n.º 3
0
 @Override
 public void reset() throws IOException {
   super.reset();
   startTerm = 0;
   nextStartOffset = 0;
   snippet = null;
   snippetBuffer.setLength(0);
   charBufferIndex = BUFFER_SIZE;
   charBufferLen = 0;
   ch = 0;
 }
Ejemplo n.º 4
0
 protected boolean getNextSnippet() throws IOException {
   startTerm = 0;
   startOffset = nextStartOffset;
   snippetBuffer.delete(0, snippetBuffer.length());
   while (true) {
     if (ch != -1) ch = readCharFromBuffer();
     if (ch == -1) break;
     else if (!isDelimiter(ch)) snippetBuffer.append((char) ch);
     else if (snippetBuffer.length() > 0) break;
     else startOffset++;
   }
   if (snippetBuffer.length() == 0) return false;
   snippet = snippetBuffer.toString();
   lenTerm = snippet.length() >= n ? n : snippet.length();
   return true;
 }
 /** {@inheritDoc} */
 @Override
 public void reset() throws IOException {
   super.reset();
   hyphenated.setLength(0);
   savedState = null;
 }