@After public void shutdownAll() throws Exception { // 1st, shutdown sorters for (UnilateralSortMerger<?> sorter : this.sorters) { if (sorter != null) { sorter.close(); } } this.sorters.clear(); // 2nd, shutdown I/O this.ioManager.shutdown(); Assert.assertTrue( "I/O Manager has not properly shut down.", this.ioManager.isProperlyShutDown()); // last, verify all memory is returned and shutdown mem manager MemoryManager memMan = getMemoryManager(); if (memMan != null) { Assert.assertTrue( "Memory Manager managed memory was not completely freed.", memMan.verifyEmpty()); memMan.shutdown(); } }
@Test public void testSortBothMerge() { try { Generator generator1 = new Generator(SEED1, INPUT_1_SIZE / 10, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH); Generator generator2 = new Generator(SEED2, INPUT_2_SIZE, 100, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH); final TestData.GeneratorIterator input1 = new TestData.GeneratorIterator(generator1, INPUT_1_SIZE); final TestData.GeneratorIterator input2 = new TestData.GeneratorIterator(generator2, INPUT_2_SIZE); final JoinFunction matcher = new NoOpMatcher(); final Collector<Record> collector = new DiscardingOutputCollector<Record>(); long start = System.nanoTime(); final UnilateralSortMerger<Record> sorter1 = new UnilateralSortMerger<Record>( this.memoryManager, this.ioManager, input1, this.parentTask, this.serializer1, this.comparator1.duplicate(), MEMORY_FOR_SORTER, 128, 0.8f); final UnilateralSortMerger<Record> sorter2 = new UnilateralSortMerger<Record>( this.memoryManager, this.ioManager, input2, this.parentTask, this.serializer2, this.comparator2.duplicate(), MEMORY_FOR_SORTER, 128, 0.8f); final MutableObjectIterator<Record> sortedInput1 = sorter1.getIterator(); final MutableObjectIterator<Record> sortedInput2 = sorter2.getIterator(); // compare with iterator values ReusingMergeInnerJoinIterator<Record, Record, Record> iterator = new ReusingMergeInnerJoinIterator<Record, Record, Record>( sortedInput1, sortedInput2, this.serializer1.getSerializer(), this.comparator1, this.serializer2.getSerializer(), this.comparator2, this.pairComparator11, this.memoryManager, this.ioManager, MEMORY_PAGES_FOR_MERGE, this.parentTask); iterator.open(); while (iterator.callWithNextKey(matcher, collector)) ; iterator.close(); sorter1.close(); sorter2.close(); long elapsed = System.nanoTime() - start; double msecs = elapsed / (1000 * 1000); System.out.println("Sort-Merge Took " + msecs + " msecs."); } catch (Exception e) { e.printStackTrace(); Assert.fail("An exception occurred during the test: " + e.getMessage()); } }
@Override public void invoke() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug(getLogString("Starting data sink operator")); } ExecutionConfig executionConfig; try { ExecutionConfig c = (ExecutionConfig) InstantiationUtil.readObjectFromConfig( getJobConfiguration(), ExecutionConfig.CONFIG_KEY, getUserCodeClassLoader()); if (c != null) { executionConfig = c; } else { LOG.warn("The execution config returned by the configuration was null"); executionConfig = new ExecutionConfig(); } } catch (IOException e) { throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e); } catch (ClassNotFoundException e) { throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e); } boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled(); try { // initialize local strategies MutableObjectIterator<IT> input1; switch (this.config.getInputLocalStrategy(0)) { case NONE: // nothing to do localStrategy = null; input1 = reader; break; case SORT: // initialize sort local strategy try { // get type comparator TypeComparatorFactory<IT> compFact = this.config.getInputComparator(0, getUserCodeClassLoader()); if (compFact == null) { throw new Exception("Missing comparator factory for local strategy on input " + 0); } // initialize sorter UnilateralSortMerger<IT> sorter = new UnilateralSortMerger<IT>( getEnvironment().getMemoryManager(), getEnvironment().getIOManager(), this.reader, this, this.inputTypeSerializerFactory, compFact.createComparator(), this.config.getRelativeMemoryInput(0), this.config.getFilehandlesInput(0), this.config.getSpillingThresholdInput(0)); this.localStrategy = sorter; input1 = sorter.getIterator(); } catch (Exception e) { throw new RuntimeException( "Initializing the input processing failed" + (e.getMessage() == null ? "." : ": " + e.getMessage()), e); } break; default: throw new RuntimeException("Invalid local strategy for DataSinkTask"); } // read the reader and write it to the output final TypeSerializer<IT> serializer = this.inputTypeSerializerFactory.getSerializer(); final MutableObjectIterator<IT> input = input1; final OutputFormat<IT> format = this.format; // check if task has been canceled if (this.taskCanceled) { return; } if (LOG.isDebugEnabled()) { LOG.debug(getLogString("Starting to produce output")); } // open format.open( this.getEnvironment().getIndexInSubtaskGroup(), this.getEnvironment().getNumberOfSubtasks()); if (objectReuseEnabled) { IT record = serializer.createInstance(); // work! while (!this.taskCanceled && ((record = input.next(record)) != null)) { format.writeRecord(record); } } else { IT record; // work! while (!this.taskCanceled && ((record = input.next()) != null)) { format.writeRecord(record); } } // close. We close here such that a regular close throwing an exception marks a task as // failed. if (!this.taskCanceled) { this.format.close(); this.format = null; } } catch (Exception ex) { // make a best effort to clean up try { if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) { cleanupCalled = true; ((CleanupWhenUnsuccessful) format).tryCleanupOnError(); } } catch (Throwable t) { LOG.error("Cleanup on error failed.", t); } ex = ExceptionInChainedStubException.exceptionUnwrap(ex); if (ex instanceof CancelTaskException) { // forward canceling exception throw ex; } // drop, if the task was canceled else if (!this.taskCanceled) { if (LOG.isErrorEnabled()) { LOG.error(getLogString("Error in user code: " + ex.getMessage()), ex); } throw ex; } } finally { if (this.format != null) { // close format, if it has not been closed, yet. // This should only be the case if we had a previous error, or were canceled. try { this.format.close(); } catch (Throwable t) { if (LOG.isWarnEnabled()) { LOG.warn(getLogString("Error closing the output format"), t); } } } // close local strategy if necessary if (localStrategy != null) { try { this.localStrategy.close(); } catch (Throwable t) { LOG.error("Error closing local strategy", t); } } RegularPactTask.clearReaders(new MutableReader[] {inputReader}); } if (!this.taskCanceled) { if (LOG.isDebugEnabled()) { LOG.debug(getLogString("Finished data sink operator")); } } else { if (LOG.isDebugEnabled()) { LOG.debug(getLogString("Data sink operator cancelled")); } } }