public boolean incrementToken() throws IOException {
if (!morphQueue.isEmpty()) {
restoreState(currentState);
setAttributesFromQueue(false);
return true;
}
while (input.incrementToken()) {
final String type = typeAtt.type();
if (KOREAN_TYPE.equals(type)) {
try {
analysisKorean(termAtt.toString());
} catch (MorphException e) {
throw new RuntimeException(e);
}
} else {
return true; // pass anything else thru
}
if (!morphQueue.isEmpty()) {
setAttributesFromQueue(true);
return true;
}
}
return false;
}
private void setAttributesFromQueue(boolean isFirst) {
final KoreanToken iw = morphQueue.removeFirst();
if (isFirst && !morphQueue.isEmpty()) {
// our queue has more elements remaining (e.g. we decompounded)
// capture state for those. We set the term attribute to be empty
// so we save lots of array copying later.
termAtt.setEmpty();
currentState = captureState();
}
termAtt.setEmpty().append(iw.getTerm());
offsetAtt.setOffset(iw.getOffset(), iw.getOffset() + iw.getLength());
morphAtt.setToken(iw);
// on the first Token we preserve incoming increment:
if (!isFirst) {
posIncrAtt.setPositionIncrement(iw.getPosInc());
}
// TODO: How to handle PositionLengthAttribute correctly?
}
