protected List<InputSplit> combineSplits(
PriorityQueue<LuceneIndexInputSplit> splits,
long maxCombinedIndexSizePerSplit,
long maxNumIndexesPerSplit) {
// now take the one-split-per-index splits and combine them into multi-index-per-split splits
List<InputSplit> combinedSplits = Lists.newLinkedList();
LuceneIndexInputSplit currentSplit = splits.poll();
while (currentSplit != null) {
while (currentSplit.getLength() < maxCombinedIndexSizePerSplit) {
LuceneIndexInputSplit nextSplit = splits.peek();
if (nextSplit == null) {
break;
}
if (currentSplit.getLength() + nextSplit.getLength() > maxCombinedIndexSizePerSplit) {
break;
}
if (currentSplit.getIndexDirs().size() >= maxNumIndexesPerSplit) {
break;
}
currentSplit.combine(nextSplit);
splits.poll();
}
combinedSplits.add(currentSplit);
currentSplit = splits.poll();
}
return combinedSplits;
}
/** sorts by length (size in bytes) */
@Override
public int compareTo(LuceneIndexInputSplit other) {
return length.compareTo(other.getLength());
}
/**
* Merge other into this split Will have no effect on other
*
* @param other the split to combine
*/
public void combine(LuceneIndexInputSplit other) {
indexDirs.addAll(other.getIndexDirs());
length += other.getLength();
}
