public ReusingBuildFirstHashMatchIterator( MutableObjectIterator<V1> firstInput, MutableObjectIterator<V2> secondInput, TypeSerializer<V1> serializer1, TypeComparator<V1> comparator1, TypeSerializer<V2> serializer2, TypeComparator<V2> comparator2, TypePairComparator<V2, V1> pairComparator, MemoryManager memManager, IOManager ioManager, AbstractInvokable ownerTask, double memoryFraction) throws MemoryAllocationException { this.memManager = memManager; this.firstInput = firstInput; this.secondInput = secondInput; this.probeSideSerializer = serializer2; this.nextBuildSideObject = serializer1.createInstance(); this.tempBuildSideRecord = serializer1.createInstance(); this.hashJoin = getHashJoin( serializer1, comparator1, serializer2, comparator2, pairComparator, memManager, ioManager, ownerTask, memoryFraction); }
@SuppressWarnings("unchecked") public <X> void setInput2(MutableObjectIterator<X> input, TypeSerializer<X> serializer) { this.input2 = input; this.serializer2 = new RuntimeSerializerFactory<X>( serializer, (Class<X>) serializer.createInstance().getClass()); }
@Override public StreamRecord<T> createInstance() { try { return new StreamRecord<T>(typeSerializer.createInstance()); } catch (Exception e) { throw new RuntimeException("Cannot instantiate StreamRecord.", e); } }
public FetchingIterator( TypeSerializer<T> serializer, MutableObjectIterator<Tuple> tupleInput, SeekableFileChannelInputView recordsInputs, TypeSerializer<Tuple> tupleSerializer, int pointerPos) { this.serializer = serializer; this.tupleInput = tupleInput; this.recordsInputs = recordsInputs; this.pointerPos = pointerPos; this.value = tupleSerializer.createInstance(); }
@Override public void run() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug( this.taskContext.formatLogString("AllReduce preprocessing done. Running Reducer code.")); } final GenericReduce<T> stub = this.taskContext.getStub(); final MutableObjectIterator<T> input = this.input; final TypeSerializer<T> serializer = this.serializer; T val1 = serializer.createInstance(); if ((val1 = input.next(val1)) == null) { return; } T val2; while (running && (val2 = input.next(serializer.createInstance())) != null) { val1 = stub.reduce(val1, val2); } this.taskContext.getOutputCollector().collect(val1); }
@Override public void run() { final MutableObjectIterator<T> input = this.input; final TypeSerializer<T> serializer = this.serializer; final SpillingBuffer buffer = this.buffer; try { T record = serializer.createInstance(); while (this.running && ((record = input.next(record)) != null)) { serializer.serialize(record, buffer); } TempBarrier.this.writingDone(); } catch (Throwable t) { TempBarrier.this.setException(t); } }
@Override public void run() throws Exception { // cache references on the stack final MutableObjectIterator<T> input = this.taskContext.getInput(0); final Collector<T> output = this.taskContext.getOutputCollector(); if (objectReuseEnabled) { T record = this.taskContext.<T>getInputSerializer(0).getSerializer().createInstance(); while (this.running && ((record = input.next(record)) != null)) { output.collect(record); } } else { T record; TypeSerializer<T> serializer = this.taskContext.<T>getInputSerializer(0).getSerializer(); while (this.running && ((record = input.next(serializer.createInstance())) != null)) { output.collect(record); } } }
private SpillingResettableIterator( Iterator<T> input, TypeSerializer<T> serializer, MemoryManager memoryManager, IOManager ioManager, List<MemorySegment> memory, boolean releaseMemOnClose) { this.memoryManager = memoryManager; this.input = input; this.instance = serializer.createInstance(); this.serializer = serializer; this.memorySegments = memory; this.releaseMemoryOnClose = releaseMemOnClose; if (LOG.isDebugEnabled()) { LOG.debug( "Creating spilling resettable iterator with " + memory.size() + " pages of memory."); } this.buffer = new SpillingBuffer( ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize()); }
@Override public void invoke() throws Exception { if (LOG.isDebugEnabled()) { LOG.debug(getLogString("Starting data sink operator")); } ExecutionConfig executionConfig; try { ExecutionConfig c = (ExecutionConfig) InstantiationUtil.readObjectFromConfig( getJobConfiguration(), ExecutionConfig.CONFIG_KEY, getUserCodeClassLoader()); if (c != null) { executionConfig = c; } else { LOG.warn("The execution config returned by the configuration was null"); executionConfig = new ExecutionConfig(); } } catch (IOException e) { throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e); } catch (ClassNotFoundException e) { throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e); } boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled(); try { // initialize local strategies MutableObjectIterator<IT> input1; switch (this.config.getInputLocalStrategy(0)) { case NONE: // nothing to do localStrategy = null; input1 = reader; break; case SORT: // initialize sort local strategy try { // get type comparator TypeComparatorFactory<IT> compFact = this.config.getInputComparator(0, getUserCodeClassLoader()); if (compFact == null) { throw new Exception("Missing comparator factory for local strategy on input " + 0); } // initialize sorter UnilateralSortMerger<IT> sorter = new UnilateralSortMerger<IT>( getEnvironment().getMemoryManager(), getEnvironment().getIOManager(), this.reader, this, this.inputTypeSerializerFactory, compFact.createComparator(), this.config.getRelativeMemoryInput(0), this.config.getFilehandlesInput(0), this.config.getSpillingThresholdInput(0)); this.localStrategy = sorter; input1 = sorter.getIterator(); } catch (Exception e) { throw new RuntimeException( "Initializing the input processing failed" + (e.getMessage() == null ? "." : ": " + e.getMessage()), e); } break; default: throw new RuntimeException("Invalid local strategy for DataSinkTask"); } // read the reader and write it to the output final TypeSerializer<IT> serializer = this.inputTypeSerializerFactory.getSerializer(); final MutableObjectIterator<IT> input = input1; final OutputFormat<IT> format = this.format; // check if task has been canceled if (this.taskCanceled) { return; } if (LOG.isDebugEnabled()) { LOG.debug(getLogString("Starting to produce output")); } // open format.open( this.getEnvironment().getIndexInSubtaskGroup(), this.getEnvironment().getNumberOfSubtasks()); if (objectReuseEnabled) { IT record = serializer.createInstance(); // work! while (!this.taskCanceled && ((record = input.next(record)) != null)) { format.writeRecord(record); } } else { IT record; // work! while (!this.taskCanceled && ((record = input.next()) != null)) { format.writeRecord(record); } } // close. We close here such that a regular close throwing an exception marks a task as // failed. if (!this.taskCanceled) { this.format.close(); this.format = null; } } catch (Exception ex) { // make a best effort to clean up try { if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) { cleanupCalled = true; ((CleanupWhenUnsuccessful) format).tryCleanupOnError(); } } catch (Throwable t) { LOG.error("Cleanup on error failed.", t); } ex = ExceptionInChainedStubException.exceptionUnwrap(ex); if (ex instanceof CancelTaskException) { // forward canceling exception throw ex; } // drop, if the task was canceled else if (!this.taskCanceled) { if (LOG.isErrorEnabled()) { LOG.error(getLogString("Error in user code: " + ex.getMessage()), ex); } throw ex; } } finally { if (this.format != null) { // close format, if it has not been closed, yet. // This should only be the case if we had a previous error, or were canceled. try { this.format.close(); } catch (Throwable t) { if (LOG.isWarnEnabled()) { LOG.warn(getLogString("Error closing the output format"), t); } } } // close local strategy if necessary if (localStrategy != null) { try { this.localStrategy.close(); } catch (Throwable t) { LOG.error("Error closing local strategy", t); } } RegularPactTask.clearReaders(new MutableReader[] {inputReader}); } if (!this.taskCanceled) { if (LOG.isDebugEnabled()) { LOG.debug(getLogString("Finished data sink operator")); } } else { if (LOG.isDebugEnabled()) { LOG.debug(getLogString("Data sink operator cancelled")); } } }