@Override
  protected void doExecute(
      final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
    final long startTime = System.currentTimeMillis();
    Set<String> indices = Sets.newHashSet();
    for (ActionRequest request : bulkRequest.requests) {
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        if (!indices.contains(indexRequest.index())) {
          indices.add(indexRequest.index());
        }
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        if (!indices.contains(deleteRequest.index())) {
          indices.add(deleteRequest.index());
        }
      }
    }

    if (autoCreateIndex) {
      final AtomicInteger counter = new AtomicInteger(indices.size());
      final AtomicBoolean failed = new AtomicBoolean();
      for (String index : indices) {
        if (!clusterService.state().metaData().hasConcreteIndex(index)) {
          createIndexAction.execute(
              new CreateIndexRequest(index).cause("auto(bulk api)"),
              new ActionListener<CreateIndexResponse>() {
                @Override
                public void onResponse(CreateIndexResponse result) {
                  if (counter.decrementAndGet() == 0) {
                    executeBulk(bulkRequest, startTime, listener);
                  }
                }

                @Override
                public void onFailure(Throwable e) {
                  if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
                    // we have the index, do it
                    if (counter.decrementAndGet() == 0) {
                      executeBulk(bulkRequest, startTime, listener);
                    }
                  } else if (failed.compareAndSet(false, true)) {
                    listener.onFailure(e);
                  }
                }
              });
        } else {
          if (counter.decrementAndGet() == 0) {
            executeBulk(bulkRequest, startTime, listener);
          }
        }
      }
    } else {
      executeBulk(bulkRequest, startTime, listener);
    }
  }
 private boolean setResponseFailureIfIndexMatches(
     AtomicArray<BulkItemResponse> responses,
     int idx,
     ActionRequest request,
     String index,
     Throwable e) {
   if (request instanceof IndexRequest) {
     IndexRequest indexRequest = (IndexRequest) request;
     if (index.equals(indexRequest.index())) {
       responses.set(
           idx,
           new BulkItemResponse(
               idx,
               "index",
               new BulkItemResponse.Failure(
                   indexRequest.index(), indexRequest.type(), indexRequest.id(), e)));
       return true;
     }
   } else if (request instanceof DeleteRequest) {
     DeleteRequest deleteRequest = (DeleteRequest) request;
     if (index.equals(deleteRequest.index())) {
       responses.set(
           idx,
           new BulkItemResponse(
               idx,
               "delete",
               new BulkItemResponse.Failure(
                   deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), e)));
       return true;
     }
   } else if (request instanceof UpdateRequest) {
     UpdateRequest updateRequest = (UpdateRequest) request;
     if (index.equals(updateRequest.index())) {
       responses.set(
           idx,
           new BulkItemResponse(
               idx,
               "update",
               new BulkItemResponse.Failure(
                   updateRequest.index(), updateRequest.type(), updateRequest.id(), e)));
       return true;
     }
   } else {
     throw new ElasticsearchException(
         "Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
   }
   return false;
 }
  private void executeBulk(
      final BulkRequest bulkRequest,
      final long startTime,
      final ActionListener<BulkResponse> listener,
      final AtomicArray<BulkItemResponse> responses) {
    final ClusterState clusterState = clusterService.state();
    // TODO use timeout to wait here if its blocked...
    clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);

    final ConcreteIndices concreteIndices =
        new ConcreteIndices(clusterState, indexNameExpressionResolver);
    MetaData metaData = clusterState.metaData();
    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof DocumentRequest) {
        DocumentRequest req = (DocumentRequest) request;

        if (addFailureIfIndexIsUnavailable(
            req, bulkRequest, responses, i, concreteIndices, metaData)) {
          continue;
        }

        String concreteIndex = concreteIndices.resolveIfAbsent(req);
        if (request instanceof IndexRequest) {
          IndexRequest indexRequest = (IndexRequest) request;
          MappingMetaData mappingMd = null;
          if (metaData.hasIndex(concreteIndex)) {
            mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type());
          }
          try {
            indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
          } catch (ElasticsearchParseException | RoutingMissingException e) {
            BulkItemResponse.Failure failure =
                new BulkItemResponse.Failure(
                    concreteIndex, indexRequest.type(), indexRequest.id(), e);
            BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
            responses.set(i, bulkItemResponse);
            // make sure the request gets never processed again
            bulkRequest.requests.set(i, null);
          }
        } else {
          concreteIndices.resolveIfAbsent(req);
          req.routing(
              clusterState
                  .metaData()
                  .resolveIndexRouting(req.parent(), req.routing(), req.index()));
        }
      }
    }

    // first, go over all the requests and create a ShardId -> Operations mapping
    Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();

    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index());
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    indexRequest.type(),
                    indexRequest.id(),
                    indexRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(deleteRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && deleteRequest.routing() == null) {
          // if routing is required, and no routing on the delete request, we need to broadcast
          // it....
          GroupShardsIterator groupShards =
              clusterService.operationRouting().broadcastDeleteShards(clusterState, concreteIndex);
          for (ShardIterator shardIt : groupShards) {
            List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
            if (list == null) {
              list = new ArrayList<>();
              requestsByShard.put(shardIt.shardId(), list);
            }
            list.add(new BulkItemRequest(i, deleteRequest));
          }
        } else {
          ShardId shardId =
              clusterService
                  .operationRouting()
                  .indexShards(
                      clusterState,
                      concreteIndex,
                      deleteRequest.type(),
                      deleteRequest.id(),
                      deleteRequest.routing())
                  .shardId();
          List<BulkItemRequest> list = requestsByShard.get(shardId);
          if (list == null) {
            list = new ArrayList<>();
            requestsByShard.put(shardId, list);
          }
          list.add(new BulkItemRequest(i, request));
        }
      } else if (request instanceof UpdateRequest) {
        UpdateRequest updateRequest = (UpdateRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(updateRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && updateRequest.routing() == null) {
          BulkItemResponse.Failure failure =
              new BulkItemResponse.Failure(
                  updateRequest.index(),
                  updateRequest.type(),
                  updateRequest.id(),
                  new IllegalArgumentException("routing is required for this item"));
          responses.set(i, new BulkItemResponse(i, updateRequest.type(), failure));
          continue;
        }
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    updateRequest.type(),
                    updateRequest.id(),
                    updateRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      }
    }

    if (requestsByShard.isEmpty()) {
      listener.onResponse(
          new BulkResponse(
              responses.toArray(new BulkItemResponse[responses.length()]),
              buildTookInMillis(startTime)));
      return;
    }

    final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
    for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
      final ShardId shardId = entry.getKey();
      final List<BulkItemRequest> requests = entry.getValue();
      BulkShardRequest bulkShardRequest =
          new BulkShardRequest(
              bulkRequest,
              shardId,
              bulkRequest.refresh(),
              requests.toArray(new BulkItemRequest[requests.size()]));
      bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
      bulkShardRequest.timeout(bulkRequest.timeout());
      shardBulkAction.execute(
          bulkShardRequest,
          new ActionListener<BulkShardResponse>() {
            @Override
            public void onResponse(BulkShardResponse bulkShardResponse) {
              for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
                // we may have no response if item failed
                if (bulkItemResponse.getResponse() != null) {
                  bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
                }
                responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              for (BulkItemRequest request : requests) {
                if (request.request() instanceof IndexRequest) {
                  IndexRequest indexRequest = (IndexRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(indexRequest.index()),
                              indexRequest.type(),
                              indexRequest.id(),
                              e)));
                } else if (request.request() instanceof DeleteRequest) {
                  DeleteRequest deleteRequest = (DeleteRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "delete",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(deleteRequest.index()),
                              deleteRequest.type(),
                              deleteRequest.id(),
                              e)));
                } else if (request.request() instanceof UpdateRequest) {
                  UpdateRequest updateRequest = (UpdateRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "update",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(updateRequest.index()),
                              updateRequest.type(),
                              updateRequest.id(),
                              e)));
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(
                  new BulkResponse(
                      responses.toArray(new BulkItemResponse[responses.length()]),
                      buildTookInMillis(startTime)));
            }
          });
    }
  }
  private void executeBulk(
      final BulkRequest bulkRequest,
      final long startTime,
      final ActionListener<BulkResponse> listener) {
    ClusterState clusterState = clusterService.state();
    for (ActionRequest request : bulkRequest.requests) {
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        indexRequest.routing(
            clusterState
                .metaData()
                .resolveIndexRouting(indexRequest.routing(), indexRequest.index()));
        indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index()));
        if (allowIdGeneration) {
          if (indexRequest.id() == null) {
            indexRequest.id(UUID.randomBase64UUID());
            // since we generate the id, change it to CREATE
            indexRequest.opType(IndexRequest.OpType.CREATE);
          }
        }
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index()));
      }
    }
    final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()];

    // first, go over all the requests and create a ShardId -> Operations mapping
    Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap();
    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        // handle routing
        MappingMetaData mappingMd =
            clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type());
        if (mappingMd != null) {
          try {
            indexRequest.processRouting(mappingMd);
          } catch (ElasticSearchException e) {
            responses[i] =
                new BulkItemResponse(
                    i,
                    indexRequest.opType().toString().toLowerCase(),
                    new BulkItemResponse.Failure(
                        indexRequest.index(),
                        indexRequest.type(),
                        indexRequest.id(),
                        e.getDetailedMessage()));
            continue;
          }
        }

        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    indexRequest.index(),
                    indexRequest.type(),
                    indexRequest.id(),
                    indexRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = Lists.newArrayList();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        MappingMetaData mappingMd =
            clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && deleteRequest.routing() == null) {
          // if routing is required, and no routing on the delete request, we need to broadcast
          // it....
          GroupShardsIterator groupShards =
              clusterService
                  .operationRouting()
                  .broadcastDeleteShards(clusterState, deleteRequest.index());
          for (ShardIterator shardIt : groupShards) {
            List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
            if (list == null) {
              list = Lists.newArrayList();
              requestsByShard.put(shardIt.shardId(), list);
            }
            list.add(new BulkItemRequest(i, request));
          }
        } else {
          ShardId shardId =
              clusterService
                  .operationRouting()
                  .deleteShards(
                      clusterState,
                      deleteRequest.index(),
                      deleteRequest.type(),
                      deleteRequest.id(),
                      deleteRequest.routing())
                  .shardId();
          List<BulkItemRequest> list = requestsByShard.get(shardId);
          if (list == null) {
            list = Lists.newArrayList();
            requestsByShard.put(shardId, list);
          }
          list.add(new BulkItemRequest(i, request));
        }
      }
    }

    if (requestsByShard.isEmpty()) {
      listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime));
      return;
    }

    final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
    for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
      final ShardId shardId = entry.getKey();
      final List<BulkItemRequest> requests = entry.getValue();
      BulkShardRequest bulkShardRequest =
          new BulkShardRequest(
              shardId.index().name(),
              shardId.id(),
              bulkRequest.refresh(),
              requests.toArray(new BulkItemRequest[requests.size()]));
      bulkShardRequest.replicationType(bulkRequest.replicationType());
      bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
      shardBulkAction.execute(
          bulkShardRequest,
          new ActionListener<BulkShardResponse>() {
            @Override
            public void onResponse(BulkShardResponse bulkShardResponse) {
              synchronized (responses) {
                for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) {
                  responses[bulkItemResponse.itemId()] = bulkItemResponse;
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              String message = ExceptionsHelper.detailedMessage(e);
              synchronized (responses) {
                for (BulkItemRequest request : requests) {
                  if (request.request() instanceof IndexRequest) {
                    IndexRequest indexRequest = (IndexRequest) request.request();
                    responses[request.id()] =
                        new BulkItemResponse(
                            request.id(),
                            indexRequest.opType().toString().toLowerCase(),
                            new BulkItemResponse.Failure(
                                indexRequest.index(),
                                indexRequest.type(),
                                indexRequest.id(),
                                message));
                  } else if (request.request() instanceof DeleteRequest) {
                    DeleteRequest deleteRequest = (DeleteRequest) request.request();
                    responses[request.id()] =
                        new BulkItemResponse(
                            request.id(),
                            "delete",
                            new BulkItemResponse.Failure(
                                deleteRequest.index(),
                                deleteRequest.type(),
                                deleteRequest.id(),
                                message));
                  }
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(
                  new BulkResponse(responses, System.currentTimeMillis() - startTime));
            }
          });
    }
  }