private void processBody(QueueingConsumer.Delivery task, BulkRequestBuilder bulkRequestBuilder)
throws Exception {
if (null == task) return;
byte[] body = task.getBody();
if (body == null) return;
// check for custom commands, which can be specified in the task header
String customCommand = null;
Map<String, Object> headers = task.getProperties().getHeaders();
if (null != headers) {
Object headerVal = headers.get("X-ES-Command");
if (null != headerVal) customCommand = headerVal.toString();
}
if (null == customCommand || customCommand.isEmpty()) {
// first, the "full bulk" script
if (bulkScript != null) {
String bodyStr = new String(body);
bulkScript.setNextVar("body", bodyStr);
String newBodyStr = (String) bulkScript.run();
if (newBodyStr == null) return;
body = newBodyStr.getBytes();
}
// second, the "doc per doc" script
if (script != null) {
processBodyPerLine(body, bulkRequestBuilder);
} else {
bulkRequestBuilder.add(body, 0, body.length, false);
}
} else {
// handle the custom command
handleCustomCommand(customCommand, task);
}
}
private void handleCustomCommand(String customCommand, QueueingConsumer.Delivery task)
throws Exception {
if (logger.isTraceEnabled()) {
logger.trace("executing custom command: [{}]", customCommand);
}
if (customCommand.equalsIgnoreCase("mapping")) {
CommandParser parser = null;
try {
parser = new CommandParser(task.getBody());
PutMappingResponse response =
client
.admin()
.indices()
.preparePutMapping(parser.getIndex())
.setType(parser.getType())
.setSource(parser.content)
.execute()
.actionGet();
} catch (IndexMissingException im) {
// if the index has not been created yet, we can should
// it with this mapping
logger.trace("index {} is missing, creating with mappin", parser.getIndex());
CreateIndexResponse res =
client
.admin()
.indices()
.prepareCreate(parser.getIndex())
.addMapping(parser.getType(), parser.content)
.execute()
.actionGet();
}
} else if (customCommand.equalsIgnoreCase("deleteByQuery")) {
CommandParser parser = null;
parser = new CommandParser(task.getBody());
if (null != parser.getIndex()) {
DeleteByQueryRequest dreq = new DeleteByQueryRequest(parser.getIndex());
if (null != parser.getType()) dreq.types(parser.getType());
if (null != parser.queryString)
dreq.query(new QueryStringQueryBuilder(parser.queryString));
else dreq.query(parser.content);
DeleteByQueryResponse response = client.deleteByQuery(dreq).actionGet();
}
} else {
logger.warn(
"unknown custom command - {} [{}]", customCommand, task.getEnvelope().getDeliveryTag());
}
}
@Override
public void run() {
while (true) {
if (closed) {
break;
}
try {
connection = connectionFactory.newConnection(rabbitAddresses);
channel = connection.createChannel();
} catch (Exception e) {
if (!closed) {
logger.warn("failed to created a connection / channel", e);
} else {
continue;
}
cleanup(0, "failed to connect");
try {
Thread.sleep(5000);
} catch (InterruptedException e1) {
// ignore, if we are closing, we will exit later
}
}
QueueingConsumer consumer = null;
// define the queue
try {
if (rabbitQueueDeclare) {
// only declare the queue if we should
channel.queueDeclare(
rabbitQueue /*queue*/,
rabbitQueueDurable /*durable*/,
false /*exclusive*/,
rabbitQueueAutoDelete /*autoDelete*/,
rabbitQueueArgs /*extra args*/);
}
if (rabbitExchangeDeclare) {
// only declare the exchange if we should
channel.exchangeDeclare(
rabbitExchange /*exchange*/, rabbitExchangeType /*type*/, rabbitExchangeDurable);
}
channel.basicQos(
rabbitQosPrefetchSize /*qos_prefetch_size*/,
rabbitQosPrefetchCount /*qos_prefetch_count*/,
false);
if (rabbitQueueBind) {
// only bind queue if we should
channel.queueBind(
rabbitQueue /*queue*/,
rabbitExchange /*exchange*/,
rabbitRoutingKey /*routingKey*/);
}
consumer = new QueueingConsumer(channel);
channel.basicConsume(rabbitQueue /*queue*/, false /*noAck*/, consumer);
} catch (Exception e) {
if (!closed) {
logger.warn("failed to create queue [{}]", e, rabbitQueue);
}
cleanup(0, "failed to create queue");
continue;
}
// now use the queue to listen for messages
while (true) {
if (closed) {
break;
}
QueueingConsumer.Delivery task;
try {
task = consumer.nextDelivery();
} catch (Exception e) {
if (!closed) {
logger.error("failed to get next message, reconnecting...", e);
}
cleanup(0, "failed to get message");
break;
}
if (task != null && task.getBody() != null) {
final List<Long> deliveryTags = Lists.newArrayList();
BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
try {
processBody(task, bulkRequestBuilder);
} catch (Exception e) {
logger.warn(
"failed to parse request for delivery tag [{}], ack'ing...",
e,
task.getEnvelope().getDeliveryTag());
try {
channel.basicAck(task.getEnvelope().getDeliveryTag(), false);
} catch (IOException e1) {
logger.warn("failed to ack [{}]", e1, task.getEnvelope().getDeliveryTag());
}
continue;
}
deliveryTags.add(task.getEnvelope().getDeliveryTag());
if (bulkRequestBuilder.numberOfActions() < bulkSize) {
// try and spin some more of those without timeout, so we have a bigger bulk (bounded
// by the bulk size)
try {
while ((task = consumer.nextDelivery(bulkTimeout.millis())) != null) {
try {
processBody(task, bulkRequestBuilder);
deliveryTags.add(task.getEnvelope().getDeliveryTag());
} catch (Throwable e) {
logger.warn(
"failed to parse request for delivery tag [{}], ack'ing...",
e,
task.getEnvelope().getDeliveryTag());
try {
channel.basicAck(task.getEnvelope().getDeliveryTag(), false);
} catch (Exception e1) {
logger.warn(
"failed to ack on failure [{}]", e1, task.getEnvelope().getDeliveryTag());
}
}
if (bulkRequestBuilder.numberOfActions() >= bulkSize) {
break;
}
}
} catch (InterruptedException e) {
if (closed) {
break;
}
} catch (ShutdownSignalException sse) {
logger.warn(
"Received a shutdown signal! initiatedByApplication: [{}], hard error: [{}]",
sse,
sse.isInitiatedByApplication(),
sse.isHardError());
if (!closed && sse.isInitiatedByApplication()) {
logger.error("failed to get next message, reconnecting...", sse);
}
cleanup(0, "failed to get message");
break;
}
}
if (logger.isTraceEnabled()) {
logger.trace(
"executing bulk with [{}] actions", bulkRequestBuilder.numberOfActions());
}
// if we have no bulk actions we might have processed custom commands, so ack them
if (ordered || bulkRequestBuilder.numberOfActions() == 0) {
try {
if (bulkRequestBuilder.numberOfActions() > 0) {
BulkResponse response = bulkRequestBuilder.execute().actionGet();
if (response.hasFailures()) {
// TODO write to exception queue?
logger.warn("failed to execute: " + response.buildFailureMessage());
}
}
} catch (Exception e) {
logger.warn("failed to execute bulk", e);
}
for (Long deliveryTag : deliveryTags) {
try {
channel.basicAck(deliveryTag, false);
} catch (Exception e1) {
logger.warn("failed to ack [{}]", e1, deliveryTag);
}
}
} else {
if (bulkRequestBuilder.numberOfActions() > 0) {
bulkRequestBuilder.execute(
new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse response) {
if (response.hasFailures()) {
// TODO write to exception queue?
logger.warn("failed to execute: " + response.buildFailureMessage());
}
for (Long deliveryTag : deliveryTags) {
try {
channel.basicAck(deliveryTag, false);
} catch (Exception e1) {
logger.warn("failed to ack [{}]", e1, deliveryTag);
}
}
}
@Override
public void onFailure(Throwable e) {
logger.warn(
"failed to execute bulk for delivery tags [{}], not ack'ing",
e,
deliveryTags);
}
});
}
}
}
}
}
cleanup(0, "closing river");
}