/**
* reset
*
* @param input
* @throws IOException
*/
public void reset(Reader input) throws IOException {
super.reset();
bufferIndex = 0;
offset = 0;
dataLen = 0;
finalOffset = 0;
ioBuffer.reset(); // make sure to reset the IO buffer!!
}
/**
* lucene 4.2x의 경우 데이터가 있으면 자소분리 후 true가 떨어지나, 여기서는 false로 떨어져 부득이하게 ioBuffer사이즈 상태로 조건변경
* (CharacterUtils.fill)
*
* @author 최일규
* @since 2014-07-11
*/
@Override
public final boolean incrementToken() throws IOException {
clearAttributes();
int length = 0;
int start = -1; // this variable is always initialized
char[] buffer = termAtt.buffer();
while (true) {
if (bufferIndex >= dataLen) {
offset += dataLen;
boolean isDecompose =
charUtils.fill(ioBuffer, jasoDecompose(input, decomposeMode, typoMode));
// 버퍼사이즈가 있으면 분석한다. (return false일때까지... 재귀호출)
if (ioBuffer.getLength() == 0) {
dataLen = 0; // so next offset += dataLen won't decrement offset
if (length > 0) {
break;
} else {
finalOffset = correctOffset(offset);
return false;
}
}
dataLen = ioBuffer.getLength();
bufferIndex = 0;
}
// use CharacterUtils here to support < 3.1 UTF-16 code unit behavior if the char based
// methods are gone
final int c = charUtils.codePointAt(ioBuffer.getBuffer(), bufferIndex, dataLen);
bufferIndex += Character.charCount(c);
// if it's a token char
if (isTokenChar(c)) {
// start of token
if (length == 0) {
assert start == -1;
start = offset + bufferIndex - 1;
// check if a supplementary could run out of bounds
} else if (length >= buffer.length - 1) {
// make sure a supplementary fits in the buffer
buffer = termAtt.resizeBuffer(2 + length);
}
// buffer it, normalized
length += Character.toChars(normalize(c), buffer, length);
if (length >= MAX_WORD_LEN) {
break;
}
} else if (length > 0) {
// return 'em
break;
}
}
termAtt.setLength(length);
assert start != -1;
offsetAtt.setOffset(correctOffset(start), finalOffset = correctOffset(start + length));
return true;
}
