protected List<InputSplit> combineSplits( PriorityQueue<LuceneIndexInputSplit> splits, long maxCombinedIndexSizePerSplit, long maxNumIndexesPerSplit) { // now take the one-split-per-index splits and combine them into multi-index-per-split splits List<InputSplit> combinedSplits = Lists.newLinkedList(); LuceneIndexInputSplit currentSplit = splits.poll(); while (currentSplit != null) { while (currentSplit.getLength() < maxCombinedIndexSizePerSplit) { LuceneIndexInputSplit nextSplit = splits.peek(); if (nextSplit == null) { break; } if (currentSplit.getLength() + nextSplit.getLength() > maxCombinedIndexSizePerSplit) { break; } if (currentSplit.getIndexDirs().size() >= maxNumIndexesPerSplit) { break; } currentSplit.combine(nextSplit); splits.poll(); } combinedSplits.add(currentSplit); currentSplit = splits.poll(); } return combinedSplits; }
/** sorts by length (size in bytes) */ @Override public int compareTo(LuceneIndexInputSplit other) { return length.compareTo(other.getLength()); }
/** * Merge other into this split Will have no effect on other * * @param other the split to combine */ public void combine(LuceneIndexInputSplit other) { indexDirs.addAll(other.getIndexDirs()); length += other.getLength(); }