@Override public void join( IntPair rec1, Tuple2<Integer, String> rec2, Collector<Tuple2<Integer, String>> out) throws Exception { final int k = rec1.getKey(); final int v = rec1.getValue(); final Integer key = rec2.f0; final String value = rec2.f1; Assert.assertTrue("Key does not match for matching IntPair Tuple combination.", k == key); Collection<TupleIntPairMatch> matches = this.toRemoveFrom.get(key); if (matches == null) { Assert.fail("Match " + key + " - " + v + ":" + value + " is unexpected."); } Assert.assertTrue( "Produced match was not contained: " + key + " - " + v + ":" + value, matches.remove(new TupleIntPairMatch(v, value))); if (matches.isEmpty()) { this.toRemoveFrom.remove(key); } }
@Test public void testSort() throws Exception { final int NUM_RECORDS = 559273; final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE; final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments); FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory); RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED); // write the records IntPair record = new IntPair(); int num = -1; do { generator.next(record); num++; } while (sorter.write(record) && num < NUM_RECORDS); QuickSort qs = new QuickSort(); qs.sort(sorter); MutableObjectIterator<IntPair> iter = sorter.getIterator(); IntPair readTarget = new IntPair(); int current = 0; int last = 0; iter.next(readTarget); // readTarget.getFieldInto(0, last); last = readTarget.getKey(); while ((readTarget = iter.next(readTarget)) != null) { current = readTarget.getKey(); final int cmp = last - current; if (cmp > 0) { Assert.fail("Next key is not larger or equal to previous key."); } int tmp = current; current = last; last = tmp; } // release the memory occupied by the buffers this.memoryManager.release(sorter.dispose()); }
@Test public void testReset() throws Exception { final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE; final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments); FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory); RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED); // write the buffer full with the first set of records IntPair record = new IntPair(); int num = -1; do { generator.next(record); num++; } while (sorter.write(record) && num < 3354624); sorter.reset(); // write a second sequence of records. since the values are of fixed length, we must be able to // write an equal number generator.reset(); // write the buffer full with the first set of records int num2 = -1; do { generator.next(record); num2++; } while (sorter.write(record) && num2 < 3354624); Assert.assertEquals( "The number of records written after the reset was not the same as before.", num, num2); // re-read the records generator.reset(); IntPair readTarget = new IntPair(); int i = 0; while (i < num) { generator.next(record); readTarget = sorter.getRecord(readTarget, i++); int rk = readTarget.getKey(); int gk = record.getKey(); int rv = readTarget.getValue(); int gv = record.getValue(); Assert.assertEquals("The re-read key is wrong", gk, rk); Assert.assertEquals("The re-read value is wrong", gv, rv); } // release the memory occupied by the buffers this.memoryManager.release(sorter.dispose()); }
@Test public void testWriteAndRead() throws Exception { final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE; final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments); FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory); RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED); // long startTime = System.currentTimeMillis(); // write the records IntPair record = new IntPair(); int num = -1; do { generator.next(record); num++; } while (sorter.write(record) && num < 3354624); // System.out.println("WRITE TIME " + (System.currentTimeMillis() - startTime)); // re-read the records generator.reset(); IntPair readTarget = new IntPair(); // startTime = System.currentTimeMillis(); int i = 0; while (i < num) { generator.next(record); readTarget = sorter.getRecord(readTarget, i++); int rk = readTarget.getKey(); int gk = record.getKey(); int rv = readTarget.getValue(); int gv = record.getValue(); if (gk != rk) { Assert.fail("The re-read key is wrong " + i); } if (gv != rv) { Assert.fail("The re-read value is wrong"); } } // System.out.println("READ TIME " + (System.currentTimeMillis() - startTime)); // System.out.println("RECORDS " + num); // release the memory occupied by the buffers this.memoryManager.release(sorter.dispose()); }
/** * The swap test fills the sort buffer and swaps all elements such that they are backwards. It * then resets the generator, goes backwards through the buffer and compares for equality. */ @Test public void testSwap() throws Exception { final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE; final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments); FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory); RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED); // write the records IntPair record = new IntPair(); int num = -1; do { generator.next(record); num++; } while (sorter.write(record) && num < 3354624); // swap the records int start = 0, end = num - 1; while (start < end) { sorter.swap(start++, end--); } // re-read the records generator.reset(); IntPair readTarget = new IntPair(); int i = num - 1; while (i >= 0) { generator.next(record); readTarget = sorter.getRecord(readTarget, i--); int rk = readTarget.getKey(); int gk = record.getKey(); int rv = readTarget.getValue(); int gv = record.getValue(); Assert.assertEquals("The re-read key is wrong", gk, rk); Assert.assertEquals("The re-read value is wrong", gv, rv); } // release the memory occupied by the buffers this.memoryManager.release(sorter.dispose()); }
@Test public void testWriteAndIterator() throws Exception { final int numSegments = MEMORY_SIZE / MEMORY_PAGE_SIZE; final List<MemorySegment> memory = this.memoryManager.allocatePages(new DummyInvokable(), numSegments); FixedLengthRecordSorter<IntPair> sorter = newSortBuffer(memory); RandomIntPairGenerator generator = new RandomIntPairGenerator(SEED); // write the records IntPair record = new IntPair(); int num = -1; do { generator.next(record); num++; } while (sorter.write(record)); // re-read the records generator.reset(); MutableObjectIterator<IntPair> iter = sorter.getIterator(); IntPair readTarget = new IntPair(); int count = 0; while ((readTarget = iter.next(readTarget)) != null) { count++; generator.next(record); int rk = readTarget.getKey(); int gk = record.getKey(); int rv = readTarget.getValue(); int gv = record.getValue(); Assert.assertEquals("The re-read key is wrong", gk, rk); Assert.assertEquals("The re-read value is wrong", gv, rv); } Assert.assertEquals("Incorrect number of records", num, count); // release the memory occupied by the buffers this.memoryManager.release(sorter.dispose()); }
@Override public int compareToReference(IntPair candidate) { return candidate.getKey() - this.reference; }
@Override public boolean equalToReference(IntPair candidate) { return this.reference == candidate.getKey(); }
@Override public void setReference(IntPair reference) { this.reference = reference.getKey(); }
@Test public void testSpillingSortWithIntermediateMergeIntPair() { try { // amount of pairs final int PAIRS = 50000000; // comparator final RandomIntPairGenerator generator = new RandomIntPairGenerator(12345678, PAIRS); final TypeSerializerFactory<IntPair> serializerFactory = new IntPairSerializer.IntPairSerializerFactory(); final TypeComparator<IntPair> comparator = new IntPairComparator(); // merge iterator LOG.debug("Initializing sortmerger..."); Sorter<IntPair> merger = new UnilateralSortMerger<IntPair>( this.memoryManager, this.ioManager, generator, this.parentTask, serializerFactory, comparator, (double) 64 / 78, 4, 0.7f); // emit data LOG.debug("Emitting data..."); // check order MutableObjectIterator<IntPair> iterator = merger.getIterator(); LOG.debug("Checking results..."); int pairsRead = 1; int nextStep = PAIRS / 20; IntPair rec1 = new IntPair(); IntPair rec2 = new IntPair(); Assert.assertTrue((rec1 = iterator.next(rec1)) != null); while ((rec2 = iterator.next(rec2)) != null) { final int k1 = rec1.getKey(); final int k2 = rec2.getKey(); pairsRead++; Assert.assertTrue(k1 - k2 <= 0); IntPair tmp = rec1; rec1 = rec2; rec2 = tmp; // log if (pairsRead == nextStep) { nextStep += PAIRS / 20; } } Assert.assertEquals("Not all pairs were read back in.", PAIRS, pairsRead); merger.close(); testSuccess = true; } catch (Exception e) { e.printStackTrace(); Assert.fail(e.getMessage()); } }