/** {@inheritDoc} */
@Override
public boolean incrementToken() throws IOException {
while (input.incrementToken()) {
char[] term = termAttribute.buffer();
int termLength = termAttribute.length();
if (termLength > 0 && term[termLength - 1] == '-') {
// a hyphenated word
// capture the state of the first token only
if (savedState == null) {
savedState = captureState();
}
hyphenated.append(term, 0, termLength - 1);
} else if (savedState == null) {
// not part of a hyphenated word.
return true;
} else {
// the final portion of a hyphenated word
hyphenated.append(term, 0, termLength);
unhyphenate();
return true;
}
}
if (savedState != null) {
// the final term ends with a hyphen
// add back the hyphen, for backwards compatibility.
hyphenated.append('-');
unhyphenate();
return true;
}
return false;
}
/** Writes the joined unhyphenated term */
private void unhyphenate() {
int endOffset = offsetAttribute.endOffset();
restoreState(savedState);
savedState = null;
char term[] = termAttribute.buffer();
int length = hyphenated.length();
if (length > termAttribute.length()) {
term = termAttribute.resizeBuffer(length);
}
hyphenated.getChars(0, length, term, 0);
termAttribute.setLength(length);
offsetAttribute.setOffset(offsetAttribute.startOffset(), endOffset);
hyphenated.setLength(0);
}
@Override
public void reset() throws IOException {
super.reset();
startTerm = 0;
nextStartOffset = 0;
snippet = null;
snippetBuffer.setLength(0);
charBufferIndex = BUFFER_SIZE;
charBufferLen = 0;
ch = 0;
}
protected boolean getNextSnippet() throws IOException {
startTerm = 0;
startOffset = nextStartOffset;
snippetBuffer.delete(0, snippetBuffer.length());
while (true) {
if (ch != -1) ch = readCharFromBuffer();
if (ch == -1) break;
else if (!isDelimiter(ch)) snippetBuffer.append((char) ch);
else if (snippetBuffer.length() > 0) break;
else startOffset++;
}
if (snippetBuffer.length() == 0) return false;
snippet = snippetBuffer.toString();
lenTerm = snippet.length() >= n ? n : snippet.length();
return true;
}
/** {@inheritDoc} */
@Override
public void reset() throws IOException {
super.reset();
hyphenated.setLength(0);
savedState = null;
}