/** Loop through and start the workers */
public void start() {
final int count = indexProcessorFig.getWorkerCount();
for (int i = 0; i < count; i++) {
startWorker();
}
}
/**
* Queue up an indexOperationMessage for multi region execution
*
* @param indexOperationMessage
*/
public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage) {
// don't try to produce something with nothing
if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
return;
}
final String jsonValue = ObjectJsonSerializer.INSTANCE.toString(indexOperationMessage);
final UUID newMessageId = UUIDGenerator.newTimeUUID();
final int expirationTimeInSeconds =
(int) TimeUnit.MILLISECONDS.toSeconds(indexProcessorFig.getIndexMessageTtl());
// write to the map in ES
esMapPersistence.putString(newMessageId.toString(), jsonValue, expirationTimeInSeconds);
// now queue up the index message
final ElasticsearchIndexEvent elasticsearchIndexEvent =
new ElasticsearchIndexEvent(queueFig.getPrimaryRegion(), newMessageId);
// send to the topic so all regions index the batch
offerTopic(elasticsearchIndexEvent);
}
private void startWorker() {
synchronized (mutex) {
Observable<List<QueueMessage>> consumer =
Observable.create(
new Observable.OnSubscribe<List<QueueMessage>>() {
@Override
public void call(final Subscriber<? super List<QueueMessage>> subscriber) {
// name our thread so it's easy to see
Thread.currentThread().setName("QueueConsumer_" + counter.incrementAndGet());
List<QueueMessage> drainList = null;
do {
try {
drainList = take();
// emit our list in it's entity to hand off to a worker pool
subscriber.onNext(drainList);
// take since we're in flight
inFlight.addAndGet(drainList.size());
} catch (Throwable t) {
final long sleepTime = indexProcessorFig.getFailureRetryTime();
logger.error(
"Failed to dequeue. Sleeping for {} milliseconds", sleepTime, t);
if (drainList != null) {
inFlight.addAndGet(-1 * drainList.size());
}
try {
Thread.sleep(sleepTime);
} catch (InterruptedException ie) {
// swallow
}
indexErrorCounter.inc();
}
} while (true);
}
}) // this won't block our read loop, just reads and proceeds
.flatMap(
sqsMessages -> {
// do this on a different schedule, and introduce concurrency with flatmap for
// faster processing
return Observable.just(sqsMessages)
.map(
messages -> {
if (messages == null || messages.size() == 0) {
// no messages came from the queue, move on
return null;
}
try {
// process the messages
List<IndexEventResult> indexEventResults =
callEventHandlers(messages);
// submit the processed messages to index producer
List<QueueMessage> messagesToAck = submitToIndex(indexEventResults);
if (messagesToAck.size() < messages.size()) {
logger.warn(
"Missing {} message(s) from index processing",
messages.size() - messagesToAck.size());
}
// ack each message if making it to this point
if (messagesToAck.size() > 0) {
ack(messagesToAck);
}
return messagesToAck;
} catch (Exception e) {
logger.error("Failed to ack messages", e);
return null;
// do not rethrow so we can process all of them
}
})
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler());
// end flatMap
},
indexProcessorFig.getEventConcurrencyFactor());
// start in the background
final Subscription subscription = consumer.subscribeOn(Schedulers.newThread()).subscribe();
subscriptions.add(subscription);
}
}
