private void startWorker() { synchronized (mutex) { Observable<List<QueueMessage>> consumer = Observable.create( new Observable.OnSubscribe<List<QueueMessage>>() { @Override public void call(final Subscriber<? super List<QueueMessage>> subscriber) { // name our thread so it's easy to see Thread.currentThread().setName("QueueConsumer_" + counter.incrementAndGet()); List<QueueMessage> drainList = null; do { try { drainList = take(); // emit our list in it's entity to hand off to a worker pool subscriber.onNext(drainList); // take since we're in flight inFlight.addAndGet(drainList.size()); } catch (Throwable t) { final long sleepTime = indexProcessorFig.getFailureRetryTime(); logger.error( "Failed to dequeue. Sleeping for {} milliseconds", sleepTime, t); if (drainList != null) { inFlight.addAndGet(-1 * drainList.size()); } try { Thread.sleep(sleepTime); } catch (InterruptedException ie) { // swallow } indexErrorCounter.inc(); } } while (true); } }) // this won't block our read loop, just reads and proceeds .flatMap( sqsMessages -> { // do this on a different schedule, and introduce concurrency with flatmap for // faster processing return Observable.just(sqsMessages) .map( messages -> { if (messages == null || messages.size() == 0) { // no messages came from the queue, move on return null; } try { // process the messages List<IndexEventResult> indexEventResults = callEventHandlers(messages); // submit the processed messages to index producer List<QueueMessage> messagesToAck = submitToIndex(indexEventResults); if (messagesToAck.size() < messages.size()) { logger.warn( "Missing {} message(s) from index processing", messages.size() - messagesToAck.size()); } // ack each message if making it to this point if (messagesToAck.size() > 0) { ack(messagesToAck); } return messagesToAck; } catch (Exception e) { logger.error("Failed to ack messages", e); return null; // do not rethrow so we can process all of them } }) .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()); // end flatMap }, indexProcessorFig.getEventConcurrencyFactor()); // start in the background final Subscription subscription = consumer.subscribeOn(Schedulers.newThread()).subscribe(); subscriptions.add(subscription); } }