@Override
public boolean incrementToken() throws IOException {
while (input.incrementToken()) {
char text[] = termAtt.termBuffer();
int termLength = termAtt.termLength();
if (!stopTable.contains(text, 0, termLength)) {
return true;
}
}
return false;
}
public final boolean incrementToken() throws IOException {
int increment = 0;
while (input.incrementToken()) {
if (!stopWords.contains(termAttr.termBuffer(), 0, termAttr.termLength())) {
posIncrAttr.setPositionIncrement(posIncrAttr.getPositionIncrement() + increment);
return true;
}
increment += posIncrAttr.getPositionIncrement();
}
return false;
}
@Override
public boolean incrementToken() throws IOException {
if (!input.incrementToken()) return false;
char[] termBuffer = termAtt.termBuffer();
int termBufferLength = termAtt.termLength();
char[] backup = null;
if (factory.maxWordCount < CapitalizationFilterFactory.DEFAULT_MAX_WORD_COUNT) {
// make a backup in case we exceed the word count
backup = new char[termBufferLength];
System.arraycopy(termBuffer, 0, backup, 0, termBufferLength);
}
if (termBufferLength < factory.maxTokenLength) {
int wordCount = 0;
int lastWordStart = 0;
for (int i = 0; i < termBufferLength; i++) {
char c = termBuffer[i];
if (c <= ' ' || c == '.') {
int len = i - lastWordStart;
if (len > 0) {
factory.processWord(termBuffer, lastWordStart, len, wordCount++);
lastWordStart = i + 1;
i++;
}
}
}
// process the last word
if (lastWordStart < termBufferLength) {
factory.processWord(
termBuffer, lastWordStart, termBufferLength - lastWordStart, wordCount++);
}
if (wordCount > factory.maxWordCount) {
termAtt.setTermBuffer(backup, 0, termBufferLength);
}
}
return true;
}