private void executeBulk( final BulkRequest bulkRequest, final long startTime, final ActionListener<BulkResponse> listener) { ClusterState clusterState = clusterService.state(); for (ActionRequest request : bulkRequest.requests) { if (request instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request; indexRequest.routing( clusterState .metaData() .resolveIndexRouting(indexRequest.routing(), indexRequest.index())); indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index())); if (allowIdGeneration) { if (indexRequest.id() == null) { indexRequest.id(UUID.randomBase64UUID()); // since we generate the id, change it to CREATE indexRequest.opType(IndexRequest.OpType.CREATE); } } } else if (request instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request; deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index())); } } final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()]; // first, go over all the requests and create a ShardId -> Operations mapping Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap(); for (int i = 0; i < bulkRequest.requests.size(); i++) { ActionRequest request = bulkRequest.requests.get(i); if (request instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request; // handle routing MappingMetaData mappingMd = clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type()); if (mappingMd != null) { try { indexRequest.processRouting(mappingMd); } catch (ElasticSearchException e) { responses[i] = new BulkItemResponse( i, indexRequest.opType().toString().toLowerCase(), new BulkItemResponse.Failure( indexRequest.index(), indexRequest.type(), indexRequest.id(), e.getDetailedMessage())); continue; } } ShardId shardId = clusterService .operationRouting() .indexShards( clusterState, indexRequest.index(), indexRequest.type(), indexRequest.id(), indexRequest.routing()) .shardId(); List<BulkItemRequest> list = requestsByShard.get(shardId); if (list == null) { list = Lists.newArrayList(); requestsByShard.put(shardId, list); } list.add(new BulkItemRequest(i, request)); } else if (request instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request; MappingMetaData mappingMd = clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type()); if (mappingMd != null && mappingMd.routing().required() && deleteRequest.routing() == null) { // if routing is required, and no routing on the delete request, we need to broadcast // it.... GroupShardsIterator groupShards = clusterService .operationRouting() .broadcastDeleteShards(clusterState, deleteRequest.index()); for (ShardIterator shardIt : groupShards) { List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId()); if (list == null) { list = Lists.newArrayList(); requestsByShard.put(shardIt.shardId(), list); } list.add(new BulkItemRequest(i, request)); } } else { ShardId shardId = clusterService .operationRouting() .deleteShards( clusterState, deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), deleteRequest.routing()) .shardId(); List<BulkItemRequest> list = requestsByShard.get(shardId); if (list == null) { list = Lists.newArrayList(); requestsByShard.put(shardId, list); } list.add(new BulkItemRequest(i, request)); } } } if (requestsByShard.isEmpty()) { listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime)); return; } final AtomicInteger counter = new AtomicInteger(requestsByShard.size()); for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) { final ShardId shardId = entry.getKey(); final List<BulkItemRequest> requests = entry.getValue(); BulkShardRequest bulkShardRequest = new BulkShardRequest( shardId.index().name(), shardId.id(), bulkRequest.refresh(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.replicationType(bulkRequest.replicationType()); bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel()); shardBulkAction.execute( bulkShardRequest, new ActionListener<BulkShardResponse>() { @Override public void onResponse(BulkShardResponse bulkShardResponse) { synchronized (responses) { for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) { responses[bulkItemResponse.itemId()] = bulkItemResponse; } } if (counter.decrementAndGet() == 0) { finishHim(); } } @Override public void onFailure(Throwable e) { // create failures for all relevant requests String message = ExceptionsHelper.detailedMessage(e); synchronized (responses) { for (BulkItemRequest request : requests) { if (request.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request.request(); responses[request.id()] = new BulkItemResponse( request.id(), indexRequest.opType().toString().toLowerCase(), new BulkItemResponse.Failure( indexRequest.index(), indexRequest.type(), indexRequest.id(), message)); } else if (request.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request.request(); responses[request.id()] = new BulkItemResponse( request.id(), "delete", new BulkItemResponse.Failure( deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), message)); } } } if (counter.decrementAndGet() == 0) { finishHim(); } } private void finishHim() { listener.onResponse( new BulkResponse(responses, System.currentTimeMillis() - startTime)); } }); } }