/** Submit results to index and return the queue messages to be ack'd */
  private List<QueueMessage> submitToIndex(List<IndexEventResult> indexEventResults) {

    // if nothing came back then return empty list
    if (indexEventResults == null) {
      return new ArrayList<>(0);
    }

    IndexOperationMessage combined = new IndexOperationMessage();
    List<QueueMessage> queueMessages =
        indexEventResults
            .stream()

            // filter out messages that are not present, they were not processed and put into the
            // results
            .filter(result -> result.getQueueMessage().isPresent())
            .map(
                indexEventResult -> {

                  // record the cycle time
                  messageCycle.update(
                      System.currentTimeMillis() - indexEventResult.getCreationTime());

                  // ingest each index op into our combined, single index op for the index producer
                  if (indexEventResult.getIndexOperationMessage().isPresent()) {
                    combined.ingest(indexEventResult.getIndexOperationMessage().get());
                  }

                  return indexEventResult.getQueueMessage().get();
                })
            // collect into a list of QueueMessages that can be ack'd later
            .collect(Collectors.toList());

    queueIndexOperationMessage(combined);

    return queueMessages;
  }
  private void startWorker() {
    synchronized (mutex) {
      Observable<List<QueueMessage>> consumer =
          Observable.create(
                  new Observable.OnSubscribe<List<QueueMessage>>() {
                    @Override
                    public void call(final Subscriber<? super List<QueueMessage>> subscriber) {

                      // name our thread so it's easy to see
                      Thread.currentThread().setName("QueueConsumer_" + counter.incrementAndGet());

                      List<QueueMessage> drainList = null;

                      do {
                        try {
                          drainList = take();
                          // emit our list in it's entity to hand off to a worker pool
                          subscriber.onNext(drainList);

                          // take since  we're in flight
                          inFlight.addAndGet(drainList.size());
                        } catch (Throwable t) {
                          final long sleepTime = indexProcessorFig.getFailureRetryTime();

                          logger.error(
                              "Failed to dequeue.  Sleeping for {} milliseconds", sleepTime, t);

                          if (drainList != null) {
                            inFlight.addAndGet(-1 * drainList.size());
                          }

                          try {
                            Thread.sleep(sleepTime);
                          } catch (InterruptedException ie) {
                            // swallow
                          }

                          indexErrorCounter.inc();
                        }
                      } while (true);
                    }
                  }) // this won't block our read loop, just reads and proceeds
              .flatMap(
                  sqsMessages -> {

                    // do this on a different schedule, and introduce concurrency with flatmap for
                    // faster processing
                    return Observable.just(sqsMessages)
                        .map(
                            messages -> {
                              if (messages == null || messages.size() == 0) {
                                // no messages came from the queue, move on
                                return null;
                              }

                              try {
                                // process the messages
                                List<IndexEventResult> indexEventResults =
                                    callEventHandlers(messages);

                                // submit the processed messages to index producer
                                List<QueueMessage> messagesToAck = submitToIndex(indexEventResults);

                                if (messagesToAck.size() < messages.size()) {
                                  logger.warn(
                                      "Missing {} message(s) from index processing",
                                      messages.size() - messagesToAck.size());
                                }

                                // ack each message if making it to this point
                                if (messagesToAck.size() > 0) {
                                  ack(messagesToAck);
                                }

                                return messagesToAck;
                              } catch (Exception e) {
                                logger.error("Failed to ack messages", e);
                                return null;
                                // do not rethrow so we can process all of them
                              }
                            })
                        .subscribeOn(rxTaskScheduler.getAsyncIOScheduler());
                    // end flatMap
                  },
                  indexProcessorFig.getEventConcurrencyFactor());

      // start in the background

      final Subscription subscription = consumer.subscribeOn(Schedulers.newThread()).subscribe();

      subscriptions.add(subscription);
    }
  }
  /**
   * calls the event handlers and returns a result with information on whether it needs to be ack'd
   * and whether it needs to be indexed
   *
   * @param messages
   * @return
   */
  private List<IndexEventResult> callEventHandlers(final List<QueueMessage> messages) {

    if (logger.isDebugEnabled()) {
      logger.debug("callEventHandlers with {} message(s)", messages.size());
    }

    Stream<IndexEventResult> indexEventResults =
        messages
            .stream()
            .map(
                message -> {
                  if (logger.isDebugEnabled()) {
                    logger.debug(
                        "Queue message with ID {} has been received {} time(s)",
                        message.getMessageId(),
                        message.getReceiveCount());
                  }

                  AsyncEvent event = null;
                  try {
                    event = (AsyncEvent) message.getBody();

                  } catch (ClassCastException cce) {
                    logger.error("Failed to deserialize message body", cce);
                    return new IndexEventResult(
                        Optional.absent(), Optional.absent(), System.currentTimeMillis());
                  }

                  if (event == null) {
                    logger.error("AsyncEvent type or event is null!");
                    return new IndexEventResult(
                        Optional.absent(), Optional.absent(), System.currentTimeMillis());
                  }

                  final AsyncEvent thisEvent = event;

                  if (logger.isDebugEnabled()) {
                    logger.debug("Processing event with type {}", event.getClass().getSimpleName());
                  }

                  try {

                    IndexOperationMessage single = new IndexOperationMessage();

                    // normal indexing event for an entity
                    if (event instanceof EntityIndexEvent) {

                      single = handleEntityIndexUpdate(message);

                    }
                    // normal indexing event for an edge
                    else if (event instanceof EdgeIndexEvent) {

                      single = handleEdgeIndex(message);

                    }
                    // deletes are 2-part, actual IO to delete data, then queue up a de-index
                    else if (event instanceof EdgeDeleteEvent) {

                      single = handleEdgeDelete(message);
                    }
                    // deletes are 2-part, actual IO to delete data, then queue up a de-index
                    else if (event instanceof EntityDeleteEvent) {

                      single = handleEntityDelete(message);
                    }
                    // initialization has special logic, therefore a special event type and no index
                    // operation message
                    else if (event instanceof InitializeApplicationIndexEvent) {

                      handleInitializeApplicationIndex(event, message);
                    }
                    // this is the main event that pulls the index doc from map persistence and
                    // hands to the index producer
                    else if (event instanceof ElasticsearchIndexEvent) {

                      handleIndexOperation((ElasticsearchIndexEvent) event);

                    } else if (event instanceof DeIndexOldVersionsEvent) {

                      single = handleDeIndexOldVersionEvent((DeIndexOldVersionsEvent) event);

                    } else {

                      throw new Exception(
                          "Unknown EventType for message: " + message.getStringBody().trim());
                    }

                    if (!(event instanceof ElasticsearchIndexEvent)
                        && !(event instanceof InitializeApplicationIndexEvent)
                        && single.isEmpty()) {
                      logger.warn(
                          "No index operation messages came back from event processing for msg: {} ",
                          message.getStringBody().trim());
                    }

                    // if no exception happens and the QueueMessage is returned in these results, it
                    // will get ack'd
                    return new IndexEventResult(
                        Optional.of(single), Optional.of(message), thisEvent.getCreationTime());

                  } catch (IndexDocNotFoundException e) {

                    // this exception is throw when we wait before trying quorum read on map
                    // persistence.
                    // return empty event result so the event's message doesn't get ack'd
                    if (logger.isDebugEnabled()) {
                      logger.debug(e.getMessage());
                    }
                    return new IndexEventResult(
                        Optional.absent(), Optional.absent(), thisEvent.getCreationTime());

                  } catch (Exception e) {

                    // NPEs don't have a detail message, so add something for our log statement to
                    // identify better
                    final String errorMessage;
                    if (e instanceof NullPointerException) {
                      errorMessage = "NullPointerException";
                    } else {
                      errorMessage = e.getMessage();
                    }

                    // if the event fails to process, log and return empty message result so it
                    // doesn't get ack'd
                    logger.error(
                        "{}. Failed to process message: {}",
                        errorMessage,
                        message.getStringBody().trim());
                    return new IndexEventResult(
                        Optional.absent(), Optional.absent(), thisEvent.getCreationTime());
                  }
                });

    return indexEventResults.collect(Collectors.toList());
  }