private void processBody(QueueingConsumer.Delivery task, BulkRequestBuilder bulkRequestBuilder) throws Exception { if (null == task) return; byte[] body = task.getBody(); if (body == null) return; // check for custom commands, which can be specified in the task header String customCommand = null; Map<String, Object> headers = task.getProperties().getHeaders(); if (null != headers) { Object headerVal = headers.get("X-ES-Command"); if (null != headerVal) customCommand = headerVal.toString(); } if (null == customCommand || customCommand.isEmpty()) { // first, the "full bulk" script if (bulkScript != null) { String bodyStr = new String(body); bulkScript.setNextVar("body", bodyStr); String newBodyStr = (String) bulkScript.run(); if (newBodyStr == null) return; body = newBodyStr.getBytes(); } // second, the "doc per doc" script if (script != null) { processBodyPerLine(body, bulkRequestBuilder); } else { bulkRequestBuilder.add(body, 0, body.length, false); } } else { // handle the custom command handleCustomCommand(customCommand, task); } }
private void handleCustomCommand(String customCommand, QueueingConsumer.Delivery task) throws Exception { if (logger.isTraceEnabled()) { logger.trace("executing custom command: [{}]", customCommand); } if (customCommand.equalsIgnoreCase("mapping")) { CommandParser parser = null; try { parser = new CommandParser(task.getBody()); PutMappingResponse response = client .admin() .indices() .preparePutMapping(parser.getIndex()) .setType(parser.getType()) .setSource(parser.content) .execute() .actionGet(); } catch (IndexMissingException im) { // if the index has not been created yet, we can should // it with this mapping logger.trace("index {} is missing, creating with mappin", parser.getIndex()); CreateIndexResponse res = client .admin() .indices() .prepareCreate(parser.getIndex()) .addMapping(parser.getType(), parser.content) .execute() .actionGet(); } } else if (customCommand.equalsIgnoreCase("deleteByQuery")) { CommandParser parser = null; parser = new CommandParser(task.getBody()); if (null != parser.getIndex()) { DeleteByQueryRequest dreq = new DeleteByQueryRequest(parser.getIndex()); if (null != parser.getType()) dreq.types(parser.getType()); if (null != parser.queryString) dreq.query(new QueryStringQueryBuilder(parser.queryString)); else dreq.query(parser.content); DeleteByQueryResponse response = client.deleteByQuery(dreq).actionGet(); } } else { logger.warn( "unknown custom command - {} [{}]", customCommand, task.getEnvelope().getDeliveryTag()); } }
@Override public void run() { while (true) { if (closed) { break; } try { connection = connectionFactory.newConnection(rabbitAddresses); channel = connection.createChannel(); } catch (Exception e) { if (!closed) { logger.warn("failed to created a connection / channel", e); } else { continue; } cleanup(0, "failed to connect"); try { Thread.sleep(5000); } catch (InterruptedException e1) { // ignore, if we are closing, we will exit later } } QueueingConsumer consumer = null; // define the queue try { if (rabbitQueueDeclare) { // only declare the queue if we should channel.queueDeclare( rabbitQueue /*queue*/, rabbitQueueDurable /*durable*/, false /*exclusive*/, rabbitQueueAutoDelete /*autoDelete*/, rabbitQueueArgs /*extra args*/); } if (rabbitExchangeDeclare) { // only declare the exchange if we should channel.exchangeDeclare( rabbitExchange /*exchange*/, rabbitExchangeType /*type*/, rabbitExchangeDurable); } channel.basicQos( rabbitQosPrefetchSize /*qos_prefetch_size*/, rabbitQosPrefetchCount /*qos_prefetch_count*/, false); if (rabbitQueueBind) { // only bind queue if we should channel.queueBind( rabbitQueue /*queue*/, rabbitExchange /*exchange*/, rabbitRoutingKey /*routingKey*/); } consumer = new QueueingConsumer(channel); channel.basicConsume(rabbitQueue /*queue*/, false /*noAck*/, consumer); } catch (Exception e) { if (!closed) { logger.warn("failed to create queue [{}]", e, rabbitQueue); } cleanup(0, "failed to create queue"); continue; } // now use the queue to listen for messages while (true) { if (closed) { break; } QueueingConsumer.Delivery task; try { task = consumer.nextDelivery(); } catch (Exception e) { if (!closed) { logger.error("failed to get next message, reconnecting...", e); } cleanup(0, "failed to get message"); break; } if (task != null && task.getBody() != null) { final List<Long> deliveryTags = Lists.newArrayList(); BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); try { processBody(task, bulkRequestBuilder); } catch (Exception e) { logger.warn( "failed to parse request for delivery tag [{}], ack'ing...", e, task.getEnvelope().getDeliveryTag()); try { channel.basicAck(task.getEnvelope().getDeliveryTag(), false); } catch (IOException e1) { logger.warn("failed to ack [{}]", e1, task.getEnvelope().getDeliveryTag()); } continue; } deliveryTags.add(task.getEnvelope().getDeliveryTag()); if (bulkRequestBuilder.numberOfActions() < bulkSize) { // try and spin some more of those without timeout, so we have a bigger bulk (bounded // by the bulk size) try { while ((task = consumer.nextDelivery(bulkTimeout.millis())) != null) { try { processBody(task, bulkRequestBuilder); deliveryTags.add(task.getEnvelope().getDeliveryTag()); } catch (Throwable e) { logger.warn( "failed to parse request for delivery tag [{}], ack'ing...", e, task.getEnvelope().getDeliveryTag()); try { channel.basicAck(task.getEnvelope().getDeliveryTag(), false); } catch (Exception e1) { logger.warn( "failed to ack on failure [{}]", e1, task.getEnvelope().getDeliveryTag()); } } if (bulkRequestBuilder.numberOfActions() >= bulkSize) { break; } } } catch (InterruptedException e) { if (closed) { break; } } catch (ShutdownSignalException sse) { logger.warn( "Received a shutdown signal! initiatedByApplication: [{}], hard error: [{}]", sse, sse.isInitiatedByApplication(), sse.isHardError()); if (!closed && sse.isInitiatedByApplication()) { logger.error("failed to get next message, reconnecting...", sse); } cleanup(0, "failed to get message"); break; } } if (logger.isTraceEnabled()) { logger.trace( "executing bulk with [{}] actions", bulkRequestBuilder.numberOfActions()); } // if we have no bulk actions we might have processed custom commands, so ack them if (ordered || bulkRequestBuilder.numberOfActions() == 0) { try { if (bulkRequestBuilder.numberOfActions() > 0) { BulkResponse response = bulkRequestBuilder.execute().actionGet(); if (response.hasFailures()) { // TODO write to exception queue? logger.warn("failed to execute: " + response.buildFailureMessage()); } } } catch (Exception e) { logger.warn("failed to execute bulk", e); } for (Long deliveryTag : deliveryTags) { try { channel.basicAck(deliveryTag, false); } catch (Exception e1) { logger.warn("failed to ack [{}]", e1, deliveryTag); } } } else { if (bulkRequestBuilder.numberOfActions() > 0) { bulkRequestBuilder.execute( new ActionListener<BulkResponse>() { @Override public void onResponse(BulkResponse response) { if (response.hasFailures()) { // TODO write to exception queue? logger.warn("failed to execute: " + response.buildFailureMessage()); } for (Long deliveryTag : deliveryTags) { try { channel.basicAck(deliveryTag, false); } catch (Exception e1) { logger.warn("failed to ack [{}]", e1, deliveryTag); } } } @Override public void onFailure(Throwable e) { logger.warn( "failed to execute bulk for delivery tags [{}], not ack'ing", e, deliveryTags); } }); } } } } } cleanup(0, "closing river"); }