Example #1
0
    public void run() {
      LOG.trace("Aggregate on the fly task started for exchangeId: {}", original.getExchangeId());

      try {
        aggregateOnTheFly();
      } catch (Throwable e) {
        if (e instanceof Exception) {
          executionException.set((Exception) e);
        } else {
          executionException.set(ObjectHelper.wrapRuntimeCamelException(e));
        }
      } finally {
        // must signal we are done so the latch can open and let the other thread continue
        // processing
        LOG.debug(
            "Signaling we are done aggregating on the fly for exchangeId: {}",
            original.getExchangeId());
        LOG.trace("Aggregate on the fly task done for exchangeId: {}", original.getExchangeId());
        aggregationOnTheFlyDone.countDown();
      }
    }
Example #2
0
  protected void doProcessParallel(
      final Exchange original,
      final AtomicExchange result,
      final Iterable<ProcessorExchangePair> pairs,
      final boolean streaming,
      final AsyncCallback callback)
      throws Exception {

    ObjectHelper.notNull(executorService, "ExecutorService", this);
    ObjectHelper.notNull(aggregateExecutorService, "AggregateExecutorService", this);

    final CompletionService<Exchange> completion;
    if (streaming) {
      // execute tasks in parallel+streaming and aggregate in the order they are finished (out of
      // order sequence)
      completion = new ExecutorCompletionService<Exchange>(executorService);
    } else {
      // execute tasks in parallel and aggregate in the order the tasks are submitted (in order
      // sequence)
      completion = new SubmitOrderedCompletionService<Exchange>(executorService);
    }

    final AtomicInteger total = new AtomicInteger(0);
    final Iterator<ProcessorExchangePair> it = pairs.iterator();

    if (it.hasNext()) {
      // when parallel then aggregate on the fly
      final AtomicBoolean running = new AtomicBoolean(true);
      final AtomicBoolean allTasksSubmitted = new AtomicBoolean();
      final CountDownLatch aggregationOnTheFlyDone = new CountDownLatch(1);
      final AtomicException executionException = new AtomicException();

      // issue task to execute in separate thread so it can aggregate on-the-fly
      // while we submit new tasks, and those tasks complete concurrently
      // this allows us to optimize work and reduce memory consumption
      final AggregateOnTheFlyTask aggregateOnTheFlyTask =
          new AggregateOnTheFlyTask(
              result,
              original,
              total,
              completion,
              running,
              aggregationOnTheFlyDone,
              allTasksSubmitted,
              executionException);
      final AtomicBoolean aggregationTaskSubmitted = new AtomicBoolean();

      LOG.trace("Starting to submit parallel tasks");

      while (it.hasNext()) {
        final ProcessorExchangePair pair = it.next();
        final Exchange subExchange = pair.getExchange();
        updateNewExchange(subExchange, total.intValue(), pairs, it);

        completion.submit(
            new Callable<Exchange>() {
              public Exchange call() throws Exception {
                // only start the aggregation task when the task is being executed to avoid staring
                // the aggregation task to early and pile up too many threads
                if (aggregationTaskSubmitted.compareAndSet(false, true)) {
                  // but only submit the task once
                  aggregateExecutorService.submit(aggregateOnTheFlyTask);
                }

                if (!running.get()) {
                  // do not start processing the task if we are not running
                  return subExchange;
                }

                try {
                  doProcessParallel(pair);
                } catch (Throwable e) {
                  subExchange.setException(e);
                }

                // Decide whether to continue with the multicast or not; similar logic to the
                // Pipeline
                Integer number = getExchangeIndex(subExchange);
                boolean continueProcessing =
                    PipelineHelper.continueProcessing(
                        subExchange, "Parallel processing failed for number " + number, LOG);
                if (stopOnException && !continueProcessing) {
                  // signal to stop running
                  running.set(false);
                  // throw caused exception
                  if (subExchange.getException() != null) {
                    // wrap in exception to explain where it failed
                    CamelExchangeException cause =
                        new CamelExchangeException(
                            "Parallel processing failed for number " + number,
                            subExchange,
                            subExchange.getException());
                    subExchange.setException(cause);
                  }
                }

                LOG.trace("Parallel processing complete for exchange: {}", subExchange);
                return subExchange;
              }
            });

        total.incrementAndGet();
      }

      // signal all tasks has been submitted
      LOG.trace("Signaling that all {} tasks has been submitted.", total.get());
      allTasksSubmitted.set(true);

      // its to hard to do parallel async routing so we let the caller thread be synchronously
      // and have it pickup the replies and do the aggregation (eg we use a latch to wait)
      // wait for aggregation to be done
      LOG.debug(
          "Waiting for on-the-fly aggregation to complete aggregating {} responses for exchangeId: {}",
          total.get(),
          original.getExchangeId());
      aggregationOnTheFlyDone.await();

      // did we fail for whatever reason, if so throw that caused exception
      if (executionException.get() != null) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Parallel processing failed due {}", executionException.get().getMessage());
        }
        throw executionException.get();
      }
    }

    // no everything is okay so we are done
    LOG.debug("Done parallel processing {} exchanges", total);
  }