private boolean setResponseFailureIfIndexMatches(
     AtomicArray<BulkItemResponse> responses,
     int idx,
     ActionRequest request,
     String index,
     Throwable e) {
   if (request instanceof IndexRequest) {
     IndexRequest indexRequest = (IndexRequest) request;
     if (index.equals(indexRequest.index())) {
       responses.set(
           idx,
           new BulkItemResponse(
               idx,
               "index",
               new BulkItemResponse.Failure(
                   indexRequest.index(), indexRequest.type(), indexRequest.id(), e)));
       return true;
     }
   } else if (request instanceof DeleteRequest) {
     DeleteRequest deleteRequest = (DeleteRequest) request;
     if (index.equals(deleteRequest.index())) {
       responses.set(
           idx,
           new BulkItemResponse(
               idx,
               "delete",
               new BulkItemResponse.Failure(
                   deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), e)));
       return true;
     }
   } else if (request instanceof UpdateRequest) {
     UpdateRequest updateRequest = (UpdateRequest) request;
     if (index.equals(updateRequest.index())) {
       responses.set(
           idx,
           new BulkItemResponse(
               idx,
               "update",
               new BulkItemResponse.Failure(
                   updateRequest.index(), updateRequest.type(), updateRequest.id(), e)));
       return true;
     }
   } else {
     throw new ElasticsearchException(
         "Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
   }
   return false;
 }
 private UpdateResult shardUpdateOperation(
     ClusterState clusterState,
     BulkShardRequest bulkShardRequest,
     UpdateRequest updateRequest,
     IndexShard indexShard) {
   UpdateHelper.Result translate = updateHelper.prepare(updateRequest, indexShard);
   switch (translate.operation()) {
     case UPSERT:
     case INDEX:
       IndexRequest indexRequest = translate.action();
       try {
         WriteResult result =
             shardIndexOperation(bulkShardRequest, indexRequest, clusterState, indexShard, false);
         return new UpdateResult(translate, indexRequest, result);
       } catch (Throwable t) {
         t = ExceptionsHelper.unwrapCause(t);
         boolean retry = false;
         if (t instanceof VersionConflictEngineException
             || (t instanceof DocumentAlreadyExistsException
                 && translate.operation() == UpdateHelper.Operation.UPSERT)) {
           retry = true;
         }
         return new UpdateResult(translate, indexRequest, retry, t, null);
       }
     case DELETE:
       DeleteRequest deleteRequest = translate.action();
       try {
         WriteResult result = shardDeleteOperation(bulkShardRequest, deleteRequest, indexShard);
         return new UpdateResult(translate, deleteRequest, result);
       } catch (Throwable t) {
         t = ExceptionsHelper.unwrapCause(t);
         boolean retry = false;
         if (t instanceof VersionConflictEngineException) {
           retry = true;
         }
         return new UpdateResult(translate, deleteRequest, retry, t, null);
       }
     case NONE:
       UpdateResponse updateResponse = translate.action();
       indexShard.indexingService().noopUpdate(updateRequest.type());
       return new UpdateResult(translate, updateResponse);
     default:
       throw new IllegalStateException("Illegal update operation " + translate.operation());
   }
 }
  @Override
  protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(
      ClusterState clusterState, PrimaryOperationRequest shardRequest) {
    final BulkShardRequest request = shardRequest.request;
    final IndexService indexService = indicesService.indexServiceSafe(request.index());
    final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());

    long[] preVersions = new long[request.items().length];
    VersionType[] preVersionTypes = new VersionType[request.items().length];
    Translog.Location location = null;
    for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
      BulkItemRequest item = request.items()[requestIndex];
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        preVersions[requestIndex] = indexRequest.version();
        preVersionTypes[requestIndex] = indexRequest.versionType();
        try {
          WriteResult<IndexResponse> result =
              shardIndexOperation(request, indexRequest, clusterState, indexShard, true);
          location = locationToSync(location, result.location);
          // add the response
          IndexResponse indexResponse = result.response();
          setResponse(
              item,
              new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
        } catch (Throwable e) {
          // rethrow the failure if we are going to retry on primary and let parent failure to
          // handle it
          if (retryPrimaryException(e)) {
            // restore updated versions...
            for (int j = 0; j < requestIndex; j++) {
              applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
            }
            throw (ElasticsearchException) e;
          }
          if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
            logger.trace(
                "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
          } else {
            logger.debug(
                "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
          }
          // if its a conflict failure, and we already executed the request on a primary (and we
          // execute it
          // again, due to primary relocation and only processing up to N bulk items when the shard
          // gets closed)
          // then just use the response we got from the successful execution
          if (item.getPrimaryResponse() != null && isConflictException(e)) {
            setResponse(item, item.getPrimaryResponse());
          } else {
            setResponse(
                item,
                new BulkItemResponse(
                    item.id(),
                    indexRequest.opType().lowercase(),
                    new BulkItemResponse.Failure(
                        request.index(), indexRequest.type(), indexRequest.id(), e)));
          }
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        preVersions[requestIndex] = deleteRequest.version();
        preVersionTypes[requestIndex] = deleteRequest.versionType();

        try {
          // add the response
          final WriteResult<DeleteResponse> writeResult =
              shardDeleteOperation(request, deleteRequest, indexShard);
          DeleteResponse deleteResponse = writeResult.response();
          location = locationToSync(location, writeResult.location);
          setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
        } catch (Throwable e) {
          // rethrow the failure if we are going to retry on primary and let parent failure to
          // handle it
          if (retryPrimaryException(e)) {
            // restore updated versions...
            for (int j = 0; j < requestIndex; j++) {
              applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
            }
            throw (ElasticsearchException) e;
          }
          if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
            logger.trace(
                "{} failed to execute bulk item (delete) {}",
                e,
                shardRequest.shardId,
                deleteRequest);
          } else {
            logger.debug(
                "{} failed to execute bulk item (delete) {}",
                e,
                shardRequest.shardId,
                deleteRequest);
          }
          // if its a conflict failure, and we already executed the request on a primary (and we
          // execute it
          // again, due to primary relocation and only processing up to N bulk items when the shard
          // gets closed)
          // then just use the response we got from the successful execution
          if (item.getPrimaryResponse() != null && isConflictException(e)) {
            setResponse(item, item.getPrimaryResponse());
          } else {
            setResponse(
                item,
                new BulkItemResponse(
                    item.id(),
                    OP_TYPE_DELETE,
                    new BulkItemResponse.Failure(
                        request.index(), deleteRequest.type(), deleteRequest.id(), e)));
          }
        }
      } else if (item.request() instanceof UpdateRequest) {
        UpdateRequest updateRequest = (UpdateRequest) item.request();
        preVersions[requestIndex] = updateRequest.version();
        preVersionTypes[requestIndex] = updateRequest.versionType();
        //  We need to do the requested retries plus the initial attempt. We don't do <
        // 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE
        for (int updateAttemptsCount = 0;
            updateAttemptsCount <= updateRequest.retryOnConflict();
            updateAttemptsCount++) {
          UpdateResult updateResult;
          try {
            updateResult = shardUpdateOperation(clusterState, request, updateRequest, indexShard);
          } catch (Throwable t) {
            updateResult = new UpdateResult(null, null, false, t, null);
          }
          if (updateResult.success()) {
            if (updateResult.writeResult != null) {
              location = locationToSync(location, updateResult.writeResult.location);
            }
            switch (updateResult.result.operation()) {
              case UPSERT:
              case INDEX:
                WriteResult<IndexResponse> result = updateResult.writeResult;
                IndexRequest indexRequest = updateResult.request();
                BytesReference indexSourceAsBytes = indexRequest.source();
                // add the response
                IndexResponse indexResponse = result.response();
                UpdateResponse updateResponse =
                    new UpdateResponse(
                        indexResponse.getShardInfo(),
                        indexResponse.getIndex(),
                        indexResponse.getType(),
                        indexResponse.getId(),
                        indexResponse.getVersion(),
                        indexResponse.isCreated());
                if (updateRequest.fields() != null && updateRequest.fields().length > 0) {
                  Tuple<XContentType, Map<String, Object>> sourceAndContent =
                      XContentHelper.convertToMap(indexSourceAsBytes, true);
                  updateResponse.setGetResult(
                      updateHelper.extractGetResult(
                          updateRequest,
                          shardRequest.request.index(),
                          indexResponse.getVersion(),
                          sourceAndContent.v2(),
                          sourceAndContent.v1(),
                          indexSourceAsBytes));
                }
                item =
                    request.items()[requestIndex] =
                        new BulkItemRequest(request.items()[requestIndex].id(), indexRequest);
                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
                break;
              case DELETE:
                WriteResult<DeleteResponse> writeResult = updateResult.writeResult;
                DeleteResponse response = writeResult.response();
                DeleteRequest deleteRequest = updateResult.request();
                updateResponse =
                    new UpdateResponse(
                        response.getShardInfo(),
                        response.getIndex(),
                        response.getType(),
                        response.getId(),
                        response.getVersion(),
                        false);
                updateResponse.setGetResult(
                    updateHelper.extractGetResult(
                        updateRequest,
                        shardRequest.request.index(),
                        response.getVersion(),
                        updateResult.result.updatedSourceAsMap(),
                        updateResult.result.updateSourceContentType(),
                        null));
                // Replace the update request to the translated delete request to execute on the
                // replica.
                item =
                    request.items()[requestIndex] =
                        new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest);
                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
                break;
              case NONE:
                setResponse(
                    item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult));
                item.setIgnoreOnReplica(); // no need to go to the replica
                break;
            }
            // NOTE: Breaking out of the retry_on_conflict loop!
            break;
          } else if (updateResult.failure()) {
            Throwable t = updateResult.error;
            if (updateResult.retry) {
              // updateAttemptCount is 0 based and marks current attempt, if it's equal to
              // retryOnConflict we are going out of the iteration
              if (updateAttemptsCount >= updateRequest.retryOnConflict()) {
                setResponse(
                    item,
                    new BulkItemResponse(
                        item.id(),
                        OP_TYPE_UPDATE,
                        new BulkItemResponse.Failure(
                            request.index(), updateRequest.type(), updateRequest.id(), t)));
              }
            } else {
              // rethrow the failure if we are going to retry on primary and let parent failure to
              // handle it
              if (retryPrimaryException(t)) {
                // restore updated versions...
                for (int j = 0; j < requestIndex; j++) {
                  applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
                }
                throw (ElasticsearchException) t;
              }
              // if its a conflict failure, and we already executed the request on a primary (and we
              // execute it
              // again, due to primary relocation and only processing up to N bulk items when the
              // shard gets closed)
              // then just use the response we got from the successful execution
              if (item.getPrimaryResponse() != null && isConflictException(t)) {
                setResponse(item, item.getPrimaryResponse());
              } else if (updateResult.result == null) {
                setResponse(
                    item,
                    new BulkItemResponse(
                        item.id(),
                        OP_TYPE_UPDATE,
                        new BulkItemResponse.Failure(
                            shardRequest.request.index(),
                            updateRequest.type(),
                            updateRequest.id(),
                            t)));
              } else {
                switch (updateResult.result.operation()) {
                  case UPSERT:
                  case INDEX:
                    IndexRequest indexRequest = updateResult.request();
                    if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
                      logger.trace(
                          "{} failed to execute bulk item (index) {}",
                          t,
                          shardRequest.shardId,
                          indexRequest);
                    } else {
                      logger.debug(
                          "{} failed to execute bulk item (index) {}",
                          t,
                          shardRequest.shardId,
                          indexRequest);
                    }
                    setResponse(
                        item,
                        new BulkItemResponse(
                            item.id(),
                            OP_TYPE_UPDATE,
                            new BulkItemResponse.Failure(
                                request.index(), indexRequest.type(), indexRequest.id(), t)));
                    break;
                  case DELETE:
                    DeleteRequest deleteRequest = updateResult.request();
                    if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
                      logger.trace(
                          "{} failed to execute bulk item (delete) {}",
                          t,
                          shardRequest.shardId,
                          deleteRequest);
                    } else {
                      logger.debug(
                          "{} failed to execute bulk item (delete) {}",
                          t,
                          shardRequest.shardId,
                          deleteRequest);
                    }
                    setResponse(
                        item,
                        new BulkItemResponse(
                            item.id(),
                            OP_TYPE_DELETE,
                            new BulkItemResponse.Failure(
                                request.index(), deleteRequest.type(), deleteRequest.id(), t)));
                    break;
                }
              }
              // NOTE: Breaking out of the retry_on_conflict loop!
              break;
            }
          }
        }
      } else {
        throw new IllegalStateException("Unexpected index operation: " + item.request());
      }

      assert item.getPrimaryResponse() != null;
      assert preVersionTypes[requestIndex] != null;
    }

    processAfter(request.refresh(), indexShard, location);
    BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
    BulkItemRequest[] items = request.items();
    for (int i = 0; i < items.length; i++) {
      responses[i] = items[i].getPrimaryResponse();
    }
    return new Tuple<>(
        new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request);
  }
  private void executeBulk(
      final BulkRequest bulkRequest,
      final long startTime,
      final ActionListener<BulkResponse> listener,
      final AtomicArray<BulkItemResponse> responses) {
    final ClusterState clusterState = clusterService.state();
    // TODO use timeout to wait here if its blocked...
    clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);

    final ConcreteIndices concreteIndices =
        new ConcreteIndices(clusterState, indexNameExpressionResolver);
    MetaData metaData = clusterState.metaData();
    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof DocumentRequest) {
        DocumentRequest req = (DocumentRequest) request;

        if (addFailureIfIndexIsUnavailable(
            req, bulkRequest, responses, i, concreteIndices, metaData)) {
          continue;
        }

        String concreteIndex = concreteIndices.resolveIfAbsent(req);
        if (request instanceof IndexRequest) {
          IndexRequest indexRequest = (IndexRequest) request;
          MappingMetaData mappingMd = null;
          if (metaData.hasIndex(concreteIndex)) {
            mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type());
          }
          try {
            indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
          } catch (ElasticsearchParseException | RoutingMissingException e) {
            BulkItemResponse.Failure failure =
                new BulkItemResponse.Failure(
                    concreteIndex, indexRequest.type(), indexRequest.id(), e);
            BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
            responses.set(i, bulkItemResponse);
            // make sure the request gets never processed again
            bulkRequest.requests.set(i, null);
          }
        } else {
          concreteIndices.resolveIfAbsent(req);
          req.routing(
              clusterState
                  .metaData()
                  .resolveIndexRouting(req.parent(), req.routing(), req.index()));
        }
      }
    }

    // first, go over all the requests and create a ShardId -> Operations mapping
    Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();

    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index());
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    indexRequest.type(),
                    indexRequest.id(),
                    indexRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(deleteRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && deleteRequest.routing() == null) {
          // if routing is required, and no routing on the delete request, we need to broadcast
          // it....
          GroupShardsIterator groupShards =
              clusterService.operationRouting().broadcastDeleteShards(clusterState, concreteIndex);
          for (ShardIterator shardIt : groupShards) {
            List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
            if (list == null) {
              list = new ArrayList<>();
              requestsByShard.put(shardIt.shardId(), list);
            }
            list.add(new BulkItemRequest(i, deleteRequest));
          }
        } else {
          ShardId shardId =
              clusterService
                  .operationRouting()
                  .indexShards(
                      clusterState,
                      concreteIndex,
                      deleteRequest.type(),
                      deleteRequest.id(),
                      deleteRequest.routing())
                  .shardId();
          List<BulkItemRequest> list = requestsByShard.get(shardId);
          if (list == null) {
            list = new ArrayList<>();
            requestsByShard.put(shardId, list);
          }
          list.add(new BulkItemRequest(i, request));
        }
      } else if (request instanceof UpdateRequest) {
        UpdateRequest updateRequest = (UpdateRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(updateRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && updateRequest.routing() == null) {
          BulkItemResponse.Failure failure =
              new BulkItemResponse.Failure(
                  updateRequest.index(),
                  updateRequest.type(),
                  updateRequest.id(),
                  new IllegalArgumentException("routing is required for this item"));
          responses.set(i, new BulkItemResponse(i, updateRequest.type(), failure));
          continue;
        }
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    updateRequest.type(),
                    updateRequest.id(),
                    updateRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      }
    }

    if (requestsByShard.isEmpty()) {
      listener.onResponse(
          new BulkResponse(
              responses.toArray(new BulkItemResponse[responses.length()]),
              buildTookInMillis(startTime)));
      return;
    }

    final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
    for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
      final ShardId shardId = entry.getKey();
      final List<BulkItemRequest> requests = entry.getValue();
      BulkShardRequest bulkShardRequest =
          new BulkShardRequest(
              bulkRequest,
              shardId,
              bulkRequest.refresh(),
              requests.toArray(new BulkItemRequest[requests.size()]));
      bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
      bulkShardRequest.timeout(bulkRequest.timeout());
      shardBulkAction.execute(
          bulkShardRequest,
          new ActionListener<BulkShardResponse>() {
            @Override
            public void onResponse(BulkShardResponse bulkShardResponse) {
              for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
                // we may have no response if item failed
                if (bulkItemResponse.getResponse() != null) {
                  bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
                }
                responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              for (BulkItemRequest request : requests) {
                if (request.request() instanceof IndexRequest) {
                  IndexRequest indexRequest = (IndexRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(indexRequest.index()),
                              indexRequest.type(),
                              indexRequest.id(),
                              e)));
                } else if (request.request() instanceof DeleteRequest) {
                  DeleteRequest deleteRequest = (DeleteRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "delete",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(deleteRequest.index()),
                              deleteRequest.type(),
                              deleteRequest.id(),
                              e)));
                } else if (request.request() instanceof UpdateRequest) {
                  UpdateRequest updateRequest = (UpdateRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "update",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(updateRequest.index()),
                              updateRequest.type(),
                              updateRequest.id(),
                              e)));
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(
                  new BulkResponse(
                      responses.toArray(new BulkItemResponse[responses.length()]),
                      buildTookInMillis(startTime)));
            }
          });
    }
  }