@Override public boolean incrementToken() throws IOException { while (input.incrementToken()) { char text[] = termAtt.termBuffer(); int termLength = termAtt.termLength(); if (!stopTable.contains(text, 0, termLength)) { return true; } } return false; }
public final boolean incrementToken() throws IOException { int increment = 0; while (input.incrementToken()) { if (!stopWords.contains(termAttr.termBuffer(), 0, termAttr.termLength())) { posIncrAttr.setPositionIncrement(posIncrAttr.getPositionIncrement() + increment); return true; } increment += posIncrAttr.getPositionIncrement(); } return false; }
@Override public boolean incrementToken() throws IOException { if (!input.incrementToken()) return false; char[] termBuffer = termAtt.termBuffer(); int termBufferLength = termAtt.termLength(); char[] backup = null; if (factory.maxWordCount < CapitalizationFilterFactory.DEFAULT_MAX_WORD_COUNT) { // make a backup in case we exceed the word count backup = new char[termBufferLength]; System.arraycopy(termBuffer, 0, backup, 0, termBufferLength); } if (termBufferLength < factory.maxTokenLength) { int wordCount = 0; int lastWordStart = 0; for (int i = 0; i < termBufferLength; i++) { char c = termBuffer[i]; if (c <= ' ' || c == '.') { int len = i - lastWordStart; if (len > 0) { factory.processWord(termBuffer, lastWordStart, len, wordCount++); lastWordStart = i + 1; i++; } } } // process the last word if (lastWordStart < termBufferLength) { factory.processWord( termBuffer, lastWordStart, termBufferLength - lastWordStart, wordCount++); } if (wordCount > factory.maxWordCount) { termAtt.setTermBuffer(backup, 0, termBufferLength); } } return true; }