コード例 #1
0
  private void startWorker() {
    synchronized (mutex) {
      Observable<List<QueueMessage>> consumer =
          Observable.create(
                  new Observable.OnSubscribe<List<QueueMessage>>() {
                    @Override
                    public void call(final Subscriber<? super List<QueueMessage>> subscriber) {

                      // name our thread so it's easy to see
                      Thread.currentThread().setName("QueueConsumer_" + counter.incrementAndGet());

                      List<QueueMessage> drainList = null;

                      do {
                        try {
                          drainList = take();
                          // emit our list in it's entity to hand off to a worker pool
                          subscriber.onNext(drainList);

                          // take since  we're in flight
                          inFlight.addAndGet(drainList.size());
                        } catch (Throwable t) {
                          final long sleepTime = indexProcessorFig.getFailureRetryTime();

                          logger.error(
                              "Failed to dequeue.  Sleeping for {} milliseconds", sleepTime, t);

                          if (drainList != null) {
                            inFlight.addAndGet(-1 * drainList.size());
                          }

                          try {
                            Thread.sleep(sleepTime);
                          } catch (InterruptedException ie) {
                            // swallow
                          }

                          indexErrorCounter.inc();
                        }
                      } while (true);
                    }
                  }) // this won't block our read loop, just reads and proceeds
              .flatMap(
                  sqsMessages -> {

                    // do this on a different schedule, and introduce concurrency with flatmap for
                    // faster processing
                    return Observable.just(sqsMessages)
                        .map(
                            messages -> {
                              if (messages == null || messages.size() == 0) {
                                // no messages came from the queue, move on
                                return null;
                              }

                              try {
                                // process the messages
                                List<IndexEventResult> indexEventResults =
                                    callEventHandlers(messages);

                                // submit the processed messages to index producer
                                List<QueueMessage> messagesToAck = submitToIndex(indexEventResults);

                                if (messagesToAck.size() < messages.size()) {
                                  logger.warn(
                                      "Missing {} message(s) from index processing",
                                      messages.size() - messagesToAck.size());
                                }

                                // ack each message if making it to this point
                                if (messagesToAck.size() > 0) {
                                  ack(messagesToAck);
                                }

                                return messagesToAck;
                              } catch (Exception e) {
                                logger.error("Failed to ack messages", e);
                                return null;
                                // do not rethrow so we can process all of them
                              }
                            })
                        .subscribeOn(rxTaskScheduler.getAsyncIOScheduler());
                    // end flatMap
                  },
                  indexProcessorFig.getEventConcurrencyFactor());

      // start in the background

      final Subscription subscription = consumer.subscribeOn(Schedulers.newThread()).subscribe();

      subscriptions.add(subscription);
    }
  }