public void run() {
LOG.trace("Aggregate on the fly task started for exchangeId: {}", original.getExchangeId());
try {
aggregateOnTheFly();
} catch (Throwable e) {
if (e instanceof Exception) {
executionException.set((Exception) e);
} else {
executionException.set(ObjectHelper.wrapRuntimeCamelException(e));
}
} finally {
// must signal we are done so the latch can open and let the other thread continue
// processing
LOG.debug(
"Signaling we are done aggregating on the fly for exchangeId: {}",
original.getExchangeId());
LOG.trace("Aggregate on the fly task done for exchangeId: {}", original.getExchangeId());
aggregationOnTheFlyDone.countDown();
}
}
protected void doProcessParallel(
final Exchange original,
final AtomicExchange result,
final Iterable<ProcessorExchangePair> pairs,
final boolean streaming,
final AsyncCallback callback)
throws Exception {
ObjectHelper.notNull(executorService, "ExecutorService", this);
ObjectHelper.notNull(aggregateExecutorService, "AggregateExecutorService", this);
final CompletionService<Exchange> completion;
if (streaming) {
// execute tasks in parallel+streaming and aggregate in the order they are finished (out of
// order sequence)
completion = new ExecutorCompletionService<Exchange>(executorService);
} else {
// execute tasks in parallel and aggregate in the order the tasks are submitted (in order
// sequence)
completion = new SubmitOrderedCompletionService<Exchange>(executorService);
}
final AtomicInteger total = new AtomicInteger(0);
final Iterator<ProcessorExchangePair> it = pairs.iterator();
if (it.hasNext()) {
// when parallel then aggregate on the fly
final AtomicBoolean running = new AtomicBoolean(true);
final AtomicBoolean allTasksSubmitted = new AtomicBoolean();
final CountDownLatch aggregationOnTheFlyDone = new CountDownLatch(1);
final AtomicException executionException = new AtomicException();
// issue task to execute in separate thread so it can aggregate on-the-fly
// while we submit new tasks, and those tasks complete concurrently
// this allows us to optimize work and reduce memory consumption
final AggregateOnTheFlyTask aggregateOnTheFlyTask =
new AggregateOnTheFlyTask(
result,
original,
total,
completion,
running,
aggregationOnTheFlyDone,
allTasksSubmitted,
executionException);
final AtomicBoolean aggregationTaskSubmitted = new AtomicBoolean();
LOG.trace("Starting to submit parallel tasks");
while (it.hasNext()) {
final ProcessorExchangePair pair = it.next();
final Exchange subExchange = pair.getExchange();
updateNewExchange(subExchange, total.intValue(), pairs, it);
completion.submit(
new Callable<Exchange>() {
public Exchange call() throws Exception {
// only start the aggregation task when the task is being executed to avoid staring
// the aggregation task to early and pile up too many threads
if (aggregationTaskSubmitted.compareAndSet(false, true)) {
// but only submit the task once
aggregateExecutorService.submit(aggregateOnTheFlyTask);
}
if (!running.get()) {
// do not start processing the task if we are not running
return subExchange;
}
try {
doProcessParallel(pair);
} catch (Throwable e) {
subExchange.setException(e);
}
// Decide whether to continue with the multicast or not; similar logic to the
// Pipeline
Integer number = getExchangeIndex(subExchange);
boolean continueProcessing =
PipelineHelper.continueProcessing(
subExchange, "Parallel processing failed for number " + number, LOG);
if (stopOnException && !continueProcessing) {
// signal to stop running
running.set(false);
// throw caused exception
if (subExchange.getException() != null) {
// wrap in exception to explain where it failed
CamelExchangeException cause =
new CamelExchangeException(
"Parallel processing failed for number " + number,
subExchange,
subExchange.getException());
subExchange.setException(cause);
}
}
LOG.trace("Parallel processing complete for exchange: {}", subExchange);
return subExchange;
}
});
total.incrementAndGet();
}
// signal all tasks has been submitted
LOG.trace("Signaling that all {} tasks has been submitted.", total.get());
allTasksSubmitted.set(true);
// its to hard to do parallel async routing so we let the caller thread be synchronously
// and have it pickup the replies and do the aggregation (eg we use a latch to wait)
// wait for aggregation to be done
LOG.debug(
"Waiting for on-the-fly aggregation to complete aggregating {} responses for exchangeId: {}",
total.get(),
original.getExchangeId());
aggregationOnTheFlyDone.await();
// did we fail for whatever reason, if so throw that caused exception
if (executionException.get() != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Parallel processing failed due {}", executionException.get().getMessage());
}
throw executionException.get();
}
}
// no everything is okay so we are done
LOG.debug("Done parallel processing {} exchanges", total);
}
