@Override
public void close() throws IOException {
super.close();
lastPosition = 0;
// Apply on each sub-TokenStream
for (PositionedTokenStream pts : positionedTokenStreams) {
if (pts == null) {
continue;
}
pts.close();
}
readQueueResetted = false;
readQueue.clear();
}
@Override
public final boolean incrementToken() throws IOException {
clearAttributes();
// Fill the queue on first call
if (!readQueueResetted) {
readQueueResetted = true;
readQueue.clear();
for (PositionedTokenStream pts : positionedTokenStreams) {
if (pts == null) {
continue;
}
// Read first token
pts.clearAttributes();
if (pts.incrementToken()) {
// PositionedTokenStream.incrementToken() initialized internal
// variables to perform proper ordering.
// Therefore we can only add it to the queue now!
readQueue.add(pts);
} // no token left (no token at all)
}
}
// Read from the first token
PositionedTokenStream toRead = readQueue.peek();
if (toRead == null) {
return false; // end of streams
}
// Look position to see if it will be increased, see usage a bit below
int pos = toRead.getPosition();
// Copy the current token attributes from the sub-TokenStream to our AttributeSource
restoreState(toRead.captureState());
// Override the PositionIncrementAttribute
this.getAttribute(PositionIncrementAttribute.class)
.setPositionIncrement(Math.max(0, pos - lastPosition));
// Prepare next read
// We did not remove the TokenStream from the queue yet,
// because if we have another token available at the same position,
// we can save a queue movement.
toRead.clearAttributes();
if (!toRead.incrementToken()) {
// No more token to read, remove from the queue
readQueue.poll();
} else {
// Check if token position changed
if (readQueue.size() > 1) {
// If yes, re-enter in the priority queue
readQueue.add(readQueue.poll());
} // Otherwise, next call will continue with the same TokenStream (less queue movements)
}
lastPosition = pos;
return true;
}
