/** * reset * * @param input * @throws IOException */ public void reset(Reader input) throws IOException { super.reset(); bufferIndex = 0; offset = 0; dataLen = 0; finalOffset = 0; ioBuffer.reset(); // make sure to reset the IO buffer!! }
/** * lucene 4.2x의 경우 데이터가 있으면 자소분리 후 true가 떨어지나, 여기서는 false로 떨어져 부득이하게 ioBuffer사이즈 상태로 조건변경 * (CharacterUtils.fill) * * @author 최일규 * @since 2014-07-11 */ @Override public final boolean incrementToken() throws IOException { clearAttributes(); int length = 0; int start = -1; // this variable is always initialized char[] buffer = termAtt.buffer(); while (true) { if (bufferIndex >= dataLen) { offset += dataLen; boolean isDecompose = charUtils.fill(ioBuffer, jasoDecompose(input, decomposeMode, typoMode)); // 버퍼사이즈가 있으면 분석한다. (return false일때까지... 재귀호출) if (ioBuffer.getLength() == 0) { dataLen = 0; // so next offset += dataLen won't decrement offset if (length > 0) { break; } else { finalOffset = correctOffset(offset); return false; } } dataLen = ioBuffer.getLength(); bufferIndex = 0; } // use CharacterUtils here to support < 3.1 UTF-16 code unit behavior if the char based // methods are gone final int c = charUtils.codePointAt(ioBuffer.getBuffer(), bufferIndex, dataLen); bufferIndex += Character.charCount(c); // if it's a token char if (isTokenChar(c)) { // start of token if (length == 0) { assert start == -1; start = offset + bufferIndex - 1; // check if a supplementary could run out of bounds } else if (length >= buffer.length - 1) { // make sure a supplementary fits in the buffer buffer = termAtt.resizeBuffer(2 + length); } // buffer it, normalized length += Character.toChars(normalize(c), buffer, length); if (length >= MAX_WORD_LEN) { break; } } else if (length > 0) { // return 'em break; } } termAtt.setLength(length); assert start != -1; offsetAtt.setOffset(correctOffset(start), finalOffset = correctOffset(start + length)); return true; }