@Override
public void join(
IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out)
throws Exception {
final int k = rec1.getKey();
final int v = rec1.getValue();
final Integer key = rec2.f0;
final String value = rec2.f1;
Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key);
Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key);
if (matches == null) {
Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected.");
}
Assert.assertTrue(
"Produced match was not contained: " + key + " - " + v + ":" + value,
matches.remove(new TupleIntPairMatch(v, value)));
if (matches.isEmpty()) {
this.toRemoveFrom.remove(key);
}
}
@Test
public void testSort() throws Exception {
final int NUM_RECORDS = 559273;
final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
final List<MemorySegment> memory =
this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
// write the records
IntPair record = new IntPair();
int num = -1;
do {
generator.next(record);
num++;
} while (sorter.write(record) && num < NUM_RECORDS);
QuickSort qs = new QuickSort();
qs.sort(sorter);
MutableObjectIterator<IntPair> iter = sorter.getIterator();
IntPair readTarget = new IntPair();
int current = 0;
int last = 0;
iter.next(readTarget);
// readTarget.getFieldInto(0, last);
last = readTarget.getKey();
while ((readTarget = iter.next(readTarget)) != null) {
current = readTarget.getKey();
final int cmp = last - current;
if (cmp > 0) {
Assert.fail("Next key is not larger or equal to previous key.");
}
int tmp = current;
current = last;
last = tmp;
}
// release the memory occupied by the buffers
this.memoryManager.release(sorter.dispose());
}
@Test
public void testReset() throws Exception {
final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
final List<MemorySegment> memory =
this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
// write the buffer full with the first set of records
IntPair record = new IntPair();
int num = -1;
do {
generator.next(record);
num++;
} while (sorter.write(record) && num < 3354624);
sorter.reset();
// write a second sequence of records. since the values are of fixed length, we must be able to
// write an equal number
generator.reset();
// write the buffer full with the first set of records
int num2 = -1;
do {
generator.next(record);
num2++;
} while (sorter.write(record) && num2 < 3354624);
Assert.assertEquals(
"The number of records written after the reset was not the same as before.", num, num2);
// re-read the records
generator.reset();
IntPair readTarget = new IntPair();
int i = 0;
while (i < num) {
generator.next(record);
readTarget = sorter.getRecord(readTarget, i++);
int rk = readTarget.getKey();
int gk = record.getKey();
int rv = readTarget.getValue();
int gv = record.getValue();
Assert.assertEquals("The re-read key is wrong", gk, rk);
Assert.assertEquals("The re-read value is wrong", gv, rv);
}
// release the memory occupied by the buffers
this.memoryManager.release(sorter.dispose());
}
@Test
public void testWriteAndRead() throws Exception {
final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
final List<MemorySegment> memory =
this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
// long startTime = System.currentTimeMillis();
// write the records
IntPair record = new IntPair();
int num = -1;
do {
generator.next(record);
num++;
} while (sorter.write(record) && num < 3354624);
// System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime));
// re-read the records
generator.reset();
IntPair readTarget = new IntPair();
// startTime = System.currentTimeMillis();
int i = 0;
while (i < num) {
generator.next(record);
readTarget = sorter.getRecord(readTarget, i++);
int rk = readTarget.getKey();
int gk = record.getKey();
int rv = readTarget.getValue();
int gv = record.getValue();
if (gk != rk) {
Assert.fail("The re-read key is wrong " + i);
}
if (gv != rv) {
Assert.fail("The re-read value is wrong");
}
}
// System.out.println("READ TIME " + (System.currentTimeMillis() - startTime));
// System.out.println("RECORDS " + num);
// release the memory occupied by the buffers
this.memoryManager.release(sorter.dispose());
}
/**
* The swap test fills the sort buffer and swaps all elements such that they are backwards. It
* then resets the generator, goes backwards through the buffer and compares for equality.
*/
@Test
public void testSwap() throws Exception {
final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
final List<MemorySegment> memory =
this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
// write the records
IntPair record = new IntPair();
int num = -1;
do {
generator.next(record);
num++;
} while (sorter.write(record) && num < 3354624);
// swap the records
int start = 0, end = num - 1;
while (start < end) {
sorter.swap(start++, end--);
}
// re-read the records
generator.reset();
IntPair readTarget = new IntPair();
int i = num - 1;
while (i >= 0) {
generator.next(record);
readTarget = sorter.getRecord(readTarget, i--);
int rk = readTarget.getKey();
int gk = record.getKey();
int rv = readTarget.getValue();
int gv = record.getValue();
Assert.assertEquals("The re-read key is wrong", gk, rk);
Assert.assertEquals("The re-read value is wrong", gv, rv);
}
// release the memory occupied by the buffers
this.memoryManager.release(sorter.dispose());
}
@Test
public void testWriteAndIterator() throws Exception {
final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE;
final List<MemorySegment> memory =
this.memoryManager.allocatePages(new DummyInvokable(), numSegments);
FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory);
RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED);
// write the records
IntPair record = new IntPair();
int num = -1;
do {
generator.next(record);
num++;
} while (sorter.write(record));
// re-read the records
generator.reset();
MutableObjectIterator<IntPair> iter = sorter.getIterator();
IntPair readTarget = new IntPair();
int count = 0;
while ((readTarget = iter.next(readTarget)) != null) {
count++;
generator.next(record);
int rk = readTarget.getKey();
int gk = record.getKey();
int rv = readTarget.getValue();
int gv = record.getValue();
Assert.assertEquals("The re-read key is wrong", gk, rk);
Assert.assertEquals("The re-read value is wrong", gv, rv);
}
Assert.assertEquals("Incorrect number of records", num, count);
// release the memory occupied by the buffers
this.memoryManager.release(sorter.dispose());
}
@Override
public int compareToReference(IntPair candidate) {
return candidate.getKey() - this.reference;
}
@Override
public boolean equalToReference(IntPair candidate) {
return this.reference == candidate.getKey();
}
@Override
public void setReference(IntPair reference) {
this.reference = reference.getKey();
}
@Test
public void testSpillingSortWithIntermediateMergeIntPair() {
try {
// amount of pairs
final int PAIRS = 50000000;
// comparator
final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS);
final TypeSerializerFactory<IntPair> serializerFactory =
new IntPairSerializer.IntPairSerializerFactory();
final TypeComparator<IntPair> comparator = new IntPairComparator();
// merge iterator
LOG.debug("Initializing sortmerger...");
Sorter<IntPair> merger =
new UnilateralSortMerger<IntPair>(
this.memoryManager,
this.ioManager,
generator,
this.parentTask,
serializerFactory,
comparator,
(double) 64 / 78,
4,
0.7f);
// emit data
LOG.debug("Emitting data...");
// check order
MutableObjectIterator<IntPair> iterator = merger.getIterator();
LOG.debug("Checking results...");
int pairsRead = 1;
int nextStep = PAIRS / 20;
IntPair rec1 = new IntPair();
IntPair rec2 = new IntPair();
Assert.assertTrue((rec1 = iterator.next(rec1)) != null);
while ((rec2 = iterator.next(rec2)) != null) {
final int k1 = rec1.getKey();
final int k2 = rec2.getKey();
pairsRead++;
Assert.assertTrue(k1 - k2 <= 0);
IntPair tmp = rec1;
rec1 = rec2;
rec2 = tmp;
// log
if (pairsRead == nextStep) {
nextStep += PAIRS / 20;
}
}
Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead);
merger.close();
testSuccess = true;
} catch (Exception e) {
e.printStackTrace();
Assert.fail(e.getMessage());
}
}
