@Test
public void testSortBothMerge() {
try {
Generator generator1 =
new Generator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
Generator generator2 =
new Generator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
final TestData.GeneratorIterator input1 =
new TestData.GeneratorIterator(generator1, INPUT_1_SIZE);
final TestData.GeneratorIterator input2 =
new TestData.GeneratorIterator(generator2, INPUT_2_SIZE);
final MatchStub matcher = new NoOpMatcher();
final Collector<PactRecord> collector = new DiscardingOutputCollector();
// reset the generators
generator1.reset();
generator2.reset();
input1.reset();
input2.reset();
// compare with iterator values
SortMergeMatchIterator<PactRecord, PactRecord, PactRecord> iterator =
new SortMergeMatchIterator<PactRecord, PactRecord, PactRecord>(
input1,
input2,
this.serializer1,
this.comparator1,
this.serializer2,
this.comparator2,
this.pairComparator11,
this.memoryManager,
this.ioManager,
MEMORY_SIZE,
64,
0.7f,
LocalStrategy.SORT_BOTH_MERGE,
this.parentTask);
long start = System.nanoTime();
iterator.open();
while (iterator.callWithNextKey(matcher, collector)) ;
iterator.close();
long elapsed = System.nanoTime() - start;
double msecs = elapsed / (1000 * 1000);
System.out.println("Sort-Merge Took " + msecs + " msecs.");
} catch (Exception e) {
e.printStackTrace();
Assert.fail("An exception occurred during the test: " + e.getMessage());
}
}
private void doTest(
TestData.GeneratorIterator buildInput,
TestData.GeneratorIterator probeInput,
Generator bgen,
Generator pgen)
throws Exception {
// collect expected data
final Map<TestData.Key, Collection<RecordMatch>> expectedFirstMatchesMap =
HashMatchIteratorITCase.matchRecordValues(
HashMatchIteratorITCase.collectRecordData(buildInput),
HashMatchIteratorITCase.collectRecordData(probeInput));
final List<Map<TestData.Key, Collection<RecordMatch>>> expectedNMatchesMapList =
new ArrayList<Map<Key, Collection<RecordMatch>>>(NUM_PROBES);
final JoinFunction[] nMatcher = new RecordMatchRemovingJoin[NUM_PROBES];
for (int i = 0; i < NUM_PROBES; i++) {
Map<TestData.Key, Collection<RecordMatch>> tmp;
expectedNMatchesMapList.add(tmp = deepCopy(expectedFirstMatchesMap));
nMatcher[i] = new RecordMatchRemovingJoin(tmp);
}
final JoinFunction firstMatcher = new RecordMatchRemovingJoin(expectedFirstMatchesMap);
final Collector<Record> collector = new DiscardingOutputCollector<Record>();
// reset the generators
bgen.reset();
pgen.reset();
buildInput.reset();
probeInput.reset();
// compare with iterator values
BuildFirstReOpenableHashMatchIterator<Record, Record, Record> iterator =
new BuildFirstReOpenableHashMatchIterator<Record, Record, Record>(
buildInput,
probeInput,
this.recordSerializer,
this.record1Comparator,
this.recordSerializer,
this.record2Comparator,
this.recordPairComparator,
this.memoryManager,
ioManager,
this.parentTask,
1.0);
iterator.open();
// do first join with both inputs
while (iterator.callWithNextKey(firstMatcher, collector)) ;
// assert that each expected match was seen for the first input
for (Entry<TestData.Key, Collection<RecordMatch>> entry : expectedFirstMatchesMap.entrySet()) {
if (!entry.getValue().isEmpty()) {
Assert.fail("Collection for key " + entry.getKey() + " is not empty");
}
}
for (int i = 0; i < NUM_PROBES; i++) {
pgen.reset();
probeInput.reset();
// prepare ..
iterator.reopenProbe(probeInput);
// .. and do second join
while (iterator.callWithNextKey(nMatcher[i], collector)) ;
// assert that each expected match was seen for the second input
for (Entry<TestData.Key, Collection<RecordMatch>> entry :
expectedNMatchesMapList.get(i).entrySet()) {
if (!entry.getValue().isEmpty()) {
Assert.fail("Collection for key " + entry.getKey() + " is not empty");
}
}
}
iterator.close();
}
