@Test
public void testSplitIntermediatesWithUniqueHost() {
List<IntermediateEntry> intermediateEntries = new ArrayList<IntermediateEntry>();
int[] pageLengths = new int[20]; // 195MB
for (int i = 0; i < pageLengths.length; i++) {
if (i < pageLengths.length - 1) {
pageLengths[i] = 10 * 1024 * 1024;
} else {
pageLengths[i] = 5 * 1024 * 1024;
}
}
long expectedTotalLength = 0;
Task.PullHost pullHost = new Task.PullHost("host", 0);
for (int i = 0; i < 20; i++) {
List<Pair<Long, Integer>> pages = new ArrayList<Pair<Long, Integer>>();
long offset = 0;
for (int j = 0; j < pageLengths.length; j++) {
pages.add(new Pair(offset, pageLengths[j]));
offset += pageLengths[j];
expectedTotalLength += pageLengths[j];
}
IntermediateEntry interm = new IntermediateEntry(i, -1, 0, pullHost);
interm.setPages(pages);
interm.setVolume(offset);
intermediateEntries.add(interm);
}
long splitVolume = 128 * 1024 * 1024;
List<List<FetchImpl>> fetches =
Repartitioner.splitOrMergeIntermediates(
null, intermediateEntries, splitVolume, 10 * 1024 * 1024);
assertEquals(32, fetches.size());
int expectedSize = 0;
Set<FetchImpl> fetchSet = TUtil.newHashSet();
for (List<FetchImpl> list : fetches) {
expectedSize += list.size();
fetchSet.addAll(list);
}
assertEquals(expectedSize, fetchSet.size());
int index = 0;
int numZeroPosFetcher = 0;
long totalLength = 0;
Set<String> uniqPullHost = new HashSet<String>();
for (List<FetchImpl> eachFetchList : fetches) {
long length = 0;
for (FetchImpl eachFetch : eachFetchList) {
if (eachFetch.getOffset() == 0) {
numZeroPosFetcher++;
}
totalLength += eachFetch.getLength();
length += eachFetch.getLength();
uniqPullHost.add(eachFetch.getPullHost().toString());
}
assertTrue(length + " should be smaller than splitVolume", length < splitVolume);
if (index < fetches.size() - 1) {
assertTrue(
length + " should be great than 100MB" + fetches.size() + "," + index,
length >= 100 * 1024 * 1024);
}
index++;
}
assertEquals(20, numZeroPosFetcher);
assertEquals(1, uniqPullHost.size());
assertEquals(expectedTotalLength, totalLength);
}
