@Test
public void testSpillingSortWithIntermediateMerge() {
try {
// amount of pairs
final int PAIRS = 10000000;
// comparator
final Comparator<TestData.Key> keyComparator = new TestData.KeyComparator();
final TestData.Generator generator =
new TestData.Generator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.FIX_LENGTH);
final MutableObjectIterator<Record> source = new TestData.GeneratorIterator(generator, PAIRS);
// merge iterator
LOG.debug("Initializing sortmerger...");
Sorter<Record> merger =
new UnilateralSortMerger<Record>(
this.memoryManager,
this.ioManager,
source,
this.parentTask,
this.pactRecordSerializer,
this.pactRecordComparator,
(double) 64 / 78,
16,
0.7f);
// emit data
LOG.debug("Emitting data...");
// check order
MutableObjectIterator<Record> iterator = merger.getIterator();
LOG.debug("Checking results...");
int pairsRead = 1;
int nextStep = PAIRS / 20;
Record rec1 = new Record();
Record rec2 = new Record();
Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
while ((rec2 = iterator.next(rec2)) != null) {
final Key k1 = rec1.getField(0, TestData.Key.class);
final Key k2 = rec2.getField(0, TestData.Key.class);
pairsRead++;
Assert.assertTrue(keyComparator.compare(k1, k2) <= 0);
Record tmp = rec1;
rec1 = rec2;
k1.setKey(k2.getKey());
rec2 = tmp;
// log
if (pairsRead == nextStep) {
nextStep += PAIRS / 20;
}
}
Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
merger.close();
testSuccess = true;
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
try {
// amount of pairs
final int PAIRS = 50000000;
// comparator
final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
final TypeSerializerFactory<IntPair> serializerFactory =
new IntPairSerializer.IntPairSerializerFactory();
final TypeComparator<IntPair> comparator = new IntPairComparator();
// merge iterator
LOG.debug("Initializing sortmerger...");
Sorter<IntPair> merger =
new UnilateralSortMerger<IntPair>(
this.memoryManager,
this.ioManager,
generator,
this.parentTask,
serializerFactory,
comparator,
(double) 64 / 78,
4,
0.7f);
// emit data
LOG.debug("Emitting data...");
// check order
MutableObjectIterator<IntPair> iterator = merger.getIterator();
LOG.debug("Checking results...");
int pairsRead = 1;
int nextStep = PAIRS / 20;
IntPair rec1 = new IntPair();
IntPair rec2 = new IntPair();
Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
while ((rec2 = iterator.next(rec2)) != null) {
final int k1 = rec1.getKey();
final int k2 = rec2.getKey();
pairsRead++;
Assert.assertTrue(k1 - k2 <= 0);
IntPair tmp = rec1;
rec1 = rec2;
rec2 = tmp;
// log
if (pairsRead == nextStep) {
nextStep += PAIRS / 20;
}
}
Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
merger.close();
testSuccess = true;
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
@Test
public void testSpillingSort() {
try {
// comparator
final Comparator<TestData.Key> keyComparator = new TestData.KeyComparator();
final TestData.Generator generator =
new TestData.Generator(
SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.CONSTANT, VAL);
final MutableObjectIterator<Record> source =
new TestData.GeneratorIterator(generator, NUM_PAIRS);
// merge iterator
LOG.debug("Initializing sortmerger...");
Sorter<Record> merger =
new UnilateralSortMerger<Record>(
this.memoryManager,
this.ioManager,
source,
this.parentTask,
this.pactRecordSerializer,
this.pactRecordComparator,
(double) 16 / 78,
64,
0.7f);
// emit data
LOG.debug("Reading and sorting data...");
// check order
MutableObjectIterator<Record> iterator = merger.getIterator();
LOG.debug("Checking results...");
int pairsEmitted = 1;
Record rec1 = new Record();
Record rec2 = new Record();
Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
while ((rec2 = iterator.next(rec2)) != null) {
final Key k1 = rec1.getField(0, TestData.Key.class);
final Key k2 = rec2.getField(0, TestData.Key.class);
pairsEmitted++;
Assert.assertTrue(keyComparator.compare(k1, k2) <= 0);
Record tmp = rec1;
rec1 = rec2;
k1.setKey(k2.getKey());
rec2 = tmp;
}
Assert.assertTrue(NUM_PAIRS == pairsEmitted);
merger.close();
testSuccess = true;
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}