public ReusingBuildFirstHashMatchIterator(
MutableObjectIterator<V1> firstInput,
MutableObjectIterator<V2> secondInput,
TypeSerializer<V1> serializer1,
TypeComparator<V1> comparator1,
TypeSerializer<V2> serializer2,
TypeComparator<V2> comparator2,
TypePairComparator<V2, V1> pairComparator,
MemoryManager memManager,
IOManager ioManager,
AbstractInvokable ownerTask,
double memoryFraction)
throws MemoryAllocationException {
this.memManager = memManager;
this.firstInput = firstInput;
this.secondInput = secondInput;
this.probeSideSerializer = serializer2;
this.nextBuildSideObject = serializer1.createInstance();
this.tempBuildSideRecord = serializer1.createInstance();
this.hashJoin =
getHashJoin(
serializer1,
comparator1,
serializer2,
comparator2,
pairComparator,
memManager,
ioManager,
ownerTask,
memoryFraction);
}
@SuppressWarnings("unchecked")
public <X> void setInput2(MutableObjectIterator<X> input, TypeSerializer<X> serializer) {
this.input2 = input;
this.serializer2 =
new RuntimeSerializerFactory<X>(
serializer, (Class<X>) serializer.createInstance().getClass());
}
@Override
public StreamRecord<T> createInstance() {
try {
return new StreamRecord<T>(typeSerializer.createInstance());
} catch (Exception e) {
throw new RuntimeException("Cannot instantiate StreamRecord.", e);
}
}
public FetchingIterator(
TypeSerializer<T> serializer,
MutableObjectIterator<Tuple> tupleInput,
SeekableFileChannelInputView recordsInputs,
TypeSerializer<Tuple> tupleSerializer,
int pointerPos) {
this.serializer = serializer;
this.tupleInput = tupleInput;
this.recordsInputs = recordsInputs;
this.pointerPos = pointerPos;
this.value = tupleSerializer.createInstance();
}
@Override
public void run() throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug(
this.taskContext.formatLogString("AllReduce preprocessing done. Running Reducer code."));
}
final GenericReduce<T> stub = this.taskContext.getStub();
final MutableObjectIterator<T> input = this.input;
final TypeSerializer<T> serializer = this.serializer;
T val1 = serializer.createInstance();
if ((val1 = input.next(val1)) == null) {
return;
}
T val2;
while (running && (val2 = input.next(serializer.createInstance())) != null) {
val1 = stub.reduce(val1, val2);
}
this.taskContext.getOutputCollector().collect(val1);
}
@Override
public void run() {
final MutableObjectIterator<T> input = this.input;
final TypeSerializer<T> serializer = this.serializer;
final SpillingBuffer buffer = this.buffer;
try {
T record = serializer.createInstance();
while (this.running && ((record = input.next(record)) != null)) {
serializer.serialize(record, buffer);
}
TempBarrier.this.writingDone();
} catch (Throwable t) {
TempBarrier.this.setException(t);
}
}
@Override
public void run() throws Exception {
// cache references on the stack
final MutableObjectIterator<T> input = this.taskContext.getInput(0);
final Collector<T> output = this.taskContext.getOutputCollector();
if (objectReuseEnabled) {
T record = this.taskContext.<T>getInputSerializer(0).getSerializer().createInstance();
while (this.running && ((record = input.next(record)) != null)) {
output.collect(record);
}
} else {
T record;
TypeSerializer<T> serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
while (this.running && ((record = input.next(serializer.createInstance())) != null)) {
output.collect(record);
}
}
}
private SpillingResettableIterator(
Iterator<T> input,
TypeSerializer<T> serializer,
MemoryManager memoryManager,
IOManager ioManager,
List<MemorySegment> memory,
boolean releaseMemOnClose) {
this.memoryManager = memoryManager;
this.input = input;
this.instance = serializer.createInstance();
this.serializer = serializer;
this.memorySegments = memory;
this.releaseMemoryOnClose = releaseMemOnClose;
if (LOG.isDebugEnabled()) {
LOG.debug(
"Creating spilling resettable iterator with " + memory.size() + " pages of memory.");
}
this.buffer =
new SpillingBuffer(
ioManager, new ListMemorySegmentSource(memory), memoryManager.getPageSize());
}
@Override
public void invoke() throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug(getLogString("Starting data sink operator"));
}
ExecutionConfig executionConfig;
try {
ExecutionConfig c =
(ExecutionConfig)
InstantiationUtil.readObjectFromConfig(
getJobConfiguration(), ExecutionConfig.CONFIG_KEY, getUserCodeClassLoader());
if (c != null) {
executionConfig = c;
} else {
LOG.warn("The execution config returned by the configuration was null");
executionConfig = new ExecutionConfig();
}
} catch (IOException e) {
throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e);
} catch (ClassNotFoundException e) {
throw new RuntimeException("Could not load ExecutionConfig from Job Configuration: " + e);
}
boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
try {
// initialize local strategies
MutableObjectIterator<IT> input1;
switch (this.config.getInputLocalStrategy(0)) {
case NONE:
// nothing to do
localStrategy = null;
input1 = reader;
break;
case SORT:
// initialize sort local strategy
try {
// get type comparator
TypeComparatorFactory<IT> compFact =
this.config.getInputComparator(0, getUserCodeClassLoader());
if (compFact == null) {
throw new Exception("Missing comparator factory for local strategy on input " + 0);
}
// initialize sorter
UnilateralSortMerger<IT> sorter =
new UnilateralSortMerger<IT>(
getEnvironment().getMemoryManager(),
getEnvironment().getIOManager(),
this.reader,
this,
this.inputTypeSerializerFactory,
compFact.createComparator(),
this.config.getRelativeMemoryInput(0),
this.config.getFilehandlesInput(0),
this.config.getSpillingThresholdInput(0));
this.localStrategy = sorter;
input1 = sorter.getIterator();
} catch (Exception e) {
throw new RuntimeException(
"Initializing the input processing failed"
+ (e.getMessage() == null ? "." : ": " + e.getMessage()),
e);
}
break;
default:
throw new RuntimeException("Invalid local strategy for DataSinkTask");
}
// read the reader and write it to the output
final TypeSerializer<IT> serializer = this.inputTypeSerializerFactory.getSerializer();
final MutableObjectIterator<IT> input = input1;
final OutputFormat<IT> format = this.format;
// check if task has been canceled
if (this.taskCanceled) {
return;
}
if (LOG.isDebugEnabled()) {
LOG.debug(getLogString("Starting to produce output"));
}
// open
format.open(
this.getEnvironment().getIndexInSubtaskGroup(),
this.getEnvironment().getNumberOfSubtasks());
if (objectReuseEnabled) {
IT record = serializer.createInstance();
// work!
while (!this.taskCanceled && ((record = input.next(record)) != null)) {
format.writeRecord(record);
}
} else {
IT record;
// work!
while (!this.taskCanceled && ((record = input.next()) != null)) {
format.writeRecord(record);
}
}
// close. We close here such that a regular close throwing an exception marks a task as
// failed.
if (!this.taskCanceled) {
this.format.close();
this.format = null;
}
} catch (Exception ex) {
// make a best effort to clean up
try {
if (!cleanupCalled && format instanceof CleanupWhenUnsuccessful) {
cleanupCalled = true;
((CleanupWhenUnsuccessful) format).tryCleanupOnError();
}
} catch (Throwable t) {
LOG.error("Cleanup on error failed.", t);
}
ex = ExceptionInChainedStubException.exceptionUnwrap(ex);
if (ex instanceof CancelTaskException) {
// forward canceling exception
throw ex;
}
// drop, if the task was canceled
else if (!this.taskCanceled) {
if (LOG.isErrorEnabled()) {
LOG.error(getLogString("Error in user code: " + ex.getMessage()), ex);
}
throw ex;
}
} finally {
if (this.format != null) {
// close format, if it has not been closed, yet.
// This should only be the case if we had a previous error, or were canceled.
try {
this.format.close();
} catch (Throwable t) {
if (LOG.isWarnEnabled()) {
LOG.warn(getLogString("Error closing the output format"), t);
}
}
}
// close local strategy if necessary
if (localStrategy != null) {
try {
this.localStrategy.close();
} catch (Throwable t) {
LOG.error("Error closing local strategy", t);
}
}
RegularPactTask.clearReaders(new MutableReader[] {inputReader});
}
if (!this.taskCanceled) {
if (LOG.isDebugEnabled()) {
LOG.debug(getLogString("Finished data sink operator"));
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug(getLogString("Data sink operator cancelled"));
}
}
}