コード例 #1
0
  private WriteResult<DeleteResponse> shardDeleteOperation(
      BulkShardRequest request, DeleteRequest deleteRequest, IndexShard indexShard) {
    Engine.Delete delete =
        indexShard.prepareDelete(
            deleteRequest.type(),
            deleteRequest.id(),
            deleteRequest.version(),
            deleteRequest.versionType(),
            Engine.Operation.Origin.PRIMARY);
    indexShard.delete(delete);
    // update the request with the version so it will go to the replicas
    deleteRequest.versionType(delete.versionType().versionTypeForReplicationAndRecovery());
    deleteRequest.version(delete.version());

    assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version());

    DeleteResponse deleteResponse =
        new DeleteResponse(
            request.index(),
            deleteRequest.type(),
            deleteRequest.id(),
            delete.version(),
            delete.found());
    return new WriteResult(deleteResponse, delete.getTranslogLocation());
  }
コード例 #2
0
  @Override
  protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
    IndexShard indexShard =
        indicesService
            .indexServiceSafe(shardRequest.request.index())
            .shardSafe(shardRequest.shardId);
    final IngestShardRequest request = shardRequest.request;
    int size = request.items().size();
    for (int i = 0; i < size; i++) {
      IngestItemRequest item = request.items().get(i);
      if (item == null) {
        continue;
      }
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        try {
          SourceToParse sourceToParse =
              SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
                  .type(indexRequest.type())
                  .id(indexRequest.id())
                  .routing(indexRequest.routing())
                  .parent(indexRequest.parent())
                  .timestamp(indexRequest.timestamp())
                  .ttl(indexRequest.ttl());

          if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
            Engine.Index index =
                indexShard
                    .prepareIndex(sourceToParse)
                    .version(indexRequest.version())
                    .origin(Engine.Operation.Origin.REPLICA);
            indexShard.index(index);
          } else {
            Engine.Create create =
                indexShard
                    .prepareCreate(sourceToParse)
                    .version(indexRequest.version())
                    .origin(Engine.Operation.Origin.REPLICA);
            indexShard.create(create);
          }
        } catch (Exception e) {
          // ignore, we are on backup
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        try {
          Engine.Delete delete =
              indexShard
                  .prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
                  .origin(Engine.Operation.Origin.REPLICA);
          indexShard.delete(delete);
        } catch (Exception e) {
          // ignore, we are on backup
        }
      }
    }
  }
コード例 #3
0
  @Override
  protected void doExecute(
      final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
    final long startTime = System.currentTimeMillis();
    Set<String> indices = Sets.newHashSet();
    for (ActionRequest request : bulkRequest.requests) {
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        if (!indices.contains(indexRequest.index())) {
          indices.add(indexRequest.index());
        }
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        if (!indices.contains(deleteRequest.index())) {
          indices.add(deleteRequest.index());
        }
      }
    }

    if (autoCreateIndex) {
      final AtomicInteger counter = new AtomicInteger(indices.size());
      final AtomicBoolean failed = new AtomicBoolean();
      for (String index : indices) {
        if (!clusterService.state().metaData().hasConcreteIndex(index)) {
          createIndexAction.execute(
              new CreateIndexRequest(index).cause("auto(bulk api)"),
              new ActionListener<CreateIndexResponse>() {
                @Override
                public void onResponse(CreateIndexResponse result) {
                  if (counter.decrementAndGet() == 0) {
                    executeBulk(bulkRequest, startTime, listener);
                  }
                }

                @Override
                public void onFailure(Throwable e) {
                  if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
                    // we have the index, do it
                    if (counter.decrementAndGet() == 0) {
                      executeBulk(bulkRequest, startTime, listener);
                    }
                  } else if (failed.compareAndSet(false, true)) {
                    listener.onFailure(e);
                  }
                }
              });
        } else {
          if (counter.decrementAndGet() == 0) {
            executeBulk(bulkRequest, startTime, listener);
          }
        }
      }
    } else {
      executeBulk(bulkRequest, startTime, listener);
    }
  }
コード例 #4
0
 private boolean setResponseFailureIfIndexMatches(
     AtomicArray<BulkItemResponse> responses,
     int idx,
     ActionRequest request,
     String index,
     Throwable e) {
   if (request instanceof IndexRequest) {
     IndexRequest indexRequest = (IndexRequest) request;
     if (index.equals(indexRequest.index())) {
       responses.set(
           idx,
           new BulkItemResponse(
               idx,
               "index",
               new BulkItemResponse.Failure(
                   indexRequest.index(), indexRequest.type(), indexRequest.id(), e)));
       return true;
     }
   } else if (request instanceof DeleteRequest) {
     DeleteRequest deleteRequest = (DeleteRequest) request;
     if (index.equals(deleteRequest.index())) {
       responses.set(
           idx,
           new BulkItemResponse(
               idx,
               "delete",
               new BulkItemResponse.Failure(
                   deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), e)));
       return true;
     }
   } else if (request instanceof UpdateRequest) {
     UpdateRequest updateRequest = (UpdateRequest) request;
     if (index.equals(updateRequest.index())) {
       responses.set(
           idx,
           new BulkItemResponse(
               idx,
               "update",
               new BulkItemResponse.Failure(
                   updateRequest.index(), updateRequest.type(), updateRequest.id(), e)));
       return true;
     }
   } else {
     throw new ElasticsearchException(
         "Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
   }
   return false;
 }
コード例 #5
0
 public DeleteRequest(DeleteRequest request) {
   super(request);
   this.type = request.type();
   this.id = request.id();
   this.routing = request.routing();
   this.refresh = request.refresh();
   this.version = request.version();
   this.versionType = request.versionType();
 }
コード例 #6
0
  @Override
  public void handleRequest(
      final RestRequest request, final RestChannel channel, final NodeClient client) {
    DeleteRequest deleteRequest =
        new DeleteRequest(request.param("index"), request.param("type"), request.param("id"));
    deleteRequest.routing(request.param("routing"));
    deleteRequest.parent(
        request.param(
            "parent")); // order is important, set it after routing, so it will set the routing
    deleteRequest.timeout(request.paramAsTime("timeout", DeleteRequest.DEFAULT_TIMEOUT));
    deleteRequest.setRefreshPolicy(request.param("refresh"));
    deleteRequest.version(RestActions.parseVersion(request));
    deleteRequest.versionType(
        VersionType.fromString(request.param("version_type"), deleteRequest.versionType()));

    String consistencyLevel = request.param("consistency");
    if (consistencyLevel != null) {
      deleteRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel));
    }

    client.delete(deleteRequest, new RestStatusToXContentListener<>(channel));
  }
コード例 #7
0
 @Override
 protected PrimaryResponse<IngestShardResponse, IngestShardRequest> shardOperationOnPrimary(
     ClusterState clusterState, PrimaryOperationRequest shardRequest) {
   final IngestShardRequest request = shardRequest.request;
   IndexShard indexShard =
       indicesService
           .indexServiceSafe(shardRequest.request.index())
           .shardSafe(shardRequest.shardId);
   int successSize = 0;
   List<IngestItemFailure> failure = newLinkedList();
   int size = request.items().size();
   long[] versions = new long[size];
   Set<Tuple<String, String>> mappingsToUpdate = newHashSet();
   for (int i = 0; i < size; i++) {
     IngestItemRequest item = request.items().get(i);
     if (item.request() instanceof IndexRequest) {
       IndexRequest indexRequest = (IndexRequest) item.request();
       Engine.IndexingOperation op = null;
       try {
         // validate, if routing is required, that we got routing
         MappingMetaData mappingMd =
             clusterState.metaData().index(request.index()).mappingOrDefault(indexRequest.type());
         if (mappingMd != null && mappingMd.routing().required()) {
           if (indexRequest.routing() == null) {
             throw new RoutingMissingException(
                 indexRequest.index(), indexRequest.type(), indexRequest.id());
           }
         }
         SourceToParse sourceToParse =
             SourceToParse.source(SourceToParse.Origin.PRIMARY, indexRequest.source())
                 .type(indexRequest.type())
                 .id(indexRequest.id())
                 .routing(indexRequest.routing())
                 .parent(indexRequest.parent())
                 .timestamp(indexRequest.timestamp())
                 .ttl(indexRequest.ttl());
         long version;
         if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
           Engine.Index index =
               indexShard
                   .prepareIndex(sourceToParse)
                   .version(indexRequest.version())
                   .versionType(indexRequest.versionType())
                   .origin(Engine.Operation.Origin.PRIMARY);
           op = index;
           indexShard.index(index);
           version = index.version();
         } else {
           Engine.Create create =
               indexShard
                   .prepareCreate(sourceToParse)
                   .version(indexRequest.version())
                   .versionType(indexRequest.versionType())
                   .origin(Engine.Operation.Origin.PRIMARY);
           op = create;
           indexShard.create(create);
           version = create.version();
         }
         versions[i] = indexRequest.version();
         // update the version on request so it will happen on the replicas
         indexRequest.version(version);
         successSize++;
       } catch (Throwable e) {
         // rethrow the failure if we are going to retry on primary and let parent failure to
         // handle it
         if (retryPrimaryException(e)) {
           // restore updated versions...
           for (int j = 0; j < i; j++) {
             applyVersion(request.items().get(j), versions[j]);
           }
           logger.error(e.getMessage(), e);
           throw new ElasticsearchException(e.getMessage());
         }
         if (e instanceof ElasticsearchException
             && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
           logger.error(
               "[{}][{}] failed to execute bulk item (index) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               indexRequest);
         } else {
           logger.error(
               "[{}][{}] failed to execute bulk item (index) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               indexRequest);
         }
         failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
         // nullify the request so it won't execute on the replicas
         request.items().set(i, null);
       } finally {
         // update mapping on master if needed, we won't update changes to the same type, since
         // once its changed, it won't have mappers added
         if (op != null && op.parsedDoc().mappingsModified()) {
           mappingsToUpdate.add(Tuple.tuple(indexRequest.index(), indexRequest.type()));
         }
       }
     } else if (item.request() instanceof DeleteRequest) {
       DeleteRequest deleteRequest = (DeleteRequest) item.request();
       try {
         Engine.Delete delete =
             indexShard
                 .prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
                 .versionType(deleteRequest.versionType())
                 .origin(Engine.Operation.Origin.PRIMARY);
         indexShard.delete(delete);
         // update the request with teh version so it will go to the replicas
         deleteRequest.version(delete.version());
         successSize++;
       } catch (Throwable e) {
         // rethrow the failure if we are going to retry on primary and let parent failure to
         // handle it
         if (retryPrimaryException(e)) {
           // restore updated versions...
           for (int j = 0; j < i; j++) {
             applyVersion(request.items().get(j), versions[j]);
           }
           logger.error(e.getMessage(), e);
           throw new ElasticsearchException(e.getMessage());
         }
         if (e instanceof ElasticsearchException
             && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
           logger.trace(
               "[{}][{}] failed to execute bulk item (delete) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               deleteRequest);
         } else {
           logger.debug(
               "[{}][{}] failed to execute bulk item (delete) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               deleteRequest);
         }
         failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
         // nullify the request so it won't execute on the replicas
         request.items().set(i, null);
       }
     }
   }
   if (!mappingsToUpdate.isEmpty()) {
     for (Tuple<String, String> mappingToUpdate : mappingsToUpdate) {
       logger.info("mapping update {} {}", mappingToUpdate.v1(), mappingToUpdate.v2());
       updateMappingOnMaster(mappingToUpdate.v1(), mappingToUpdate.v2());
     }
   }
   IngestShardResponse response =
       new IngestShardResponse(
           new ShardId(request.index(), request.shardId()), successSize, failure);
   return new PrimaryResponse<IngestShardResponse, IngestShardRequest>(
       shardRequest.request, response, null);
 }
コード例 #8
0
  @Override
  protected void shardOperationOnReplica(ShardId shardId, BulkShardRequest request) {
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardId.id());
    Translog.Location location = null;
    for (int i = 0; i < request.items().length; i++) {
      BulkItemRequest item = request.items()[i];
      if (item == null || item.isIgnoreOnReplica()) {
        continue;
      }
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        try {
          SourceToParse sourceToParse =
              SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
                  .index(shardId.getIndex())
                  .type(indexRequest.type())
                  .id(indexRequest.id())
                  .routing(indexRequest.routing())
                  .parent(indexRequest.parent())
                  .timestamp(indexRequest.timestamp())
                  .ttl(indexRequest.ttl());

          final Engine.IndexingOperation operation;
          if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
            operation =
                indexShard.prepareIndex(
                    sourceToParse,
                    indexRequest.version(),
                    indexRequest.versionType(),
                    Engine.Operation.Origin.REPLICA,
                    request.canHaveDuplicates() || indexRequest.canHaveDuplicates());
          } else {
            assert indexRequest.opType() == IndexRequest.OpType.CREATE : indexRequest.opType();
            operation =
                indexShard.prepareCreate(
                    sourceToParse,
                    indexRequest.version(),
                    indexRequest.versionType(),
                    Engine.Operation.Origin.REPLICA,
                    request.canHaveDuplicates() || indexRequest.canHaveDuplicates(),
                    indexRequest.autoGeneratedId());
          }
          Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
          if (update != null) {
            throw new RetryOnReplicaException(
                shardId,
                "Mappings are not available on the replica yet, triggered update: " + update);
          }
          operation.execute(indexShard);
          location = locationToSync(location, operation.getTranslogLocation());
        } catch (Throwable e) {
          // if its not an ignore replica failure, we need to make sure to bubble up the failure
          // so we will fail the shard
          if (!ignoreReplicaException(e)) {
            throw e;
          }
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        try {
          Engine.Delete delete =
              indexShard.prepareDelete(
                  deleteRequest.type(),
                  deleteRequest.id(),
                  deleteRequest.version(),
                  deleteRequest.versionType(),
                  Engine.Operation.Origin.REPLICA);
          indexShard.delete(delete);
          location = locationToSync(location, delete.getTranslogLocation());
        } catch (Throwable e) {
          // if its not an ignore replica failure, we need to make sure to bubble up the failure
          // so we will fail the shard
          if (!ignoreReplicaException(e)) {
            throw e;
          }
        }
      } else {
        throw new IllegalStateException("Unexpected index operation: " + item.request());
      }
    }

    processAfter(request.refresh(), indexShard, location);
  }
コード例 #9
0
  @Override
  protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(
      ClusterState clusterState, PrimaryOperationRequest shardRequest) {
    final BulkShardRequest request = shardRequest.request;
    final IndexService indexService = indicesService.indexServiceSafe(request.index());
    final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());

    long[] preVersions = new long[request.items().length];
    VersionType[] preVersionTypes = new VersionType[request.items().length];
    Translog.Location location = null;
    for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
      BulkItemRequest item = request.items()[requestIndex];
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        preVersions[requestIndex] = indexRequest.version();
        preVersionTypes[requestIndex] = indexRequest.versionType();
        try {
          WriteResult<IndexResponse> result =
              shardIndexOperation(request, indexRequest, clusterState, indexShard, true);
          location = locationToSync(location, result.location);
          // add the response
          IndexResponse indexResponse = result.response();
          setResponse(
              item,
              new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
        } catch (Throwable e) {
          // rethrow the failure if we are going to retry on primary and let parent failure to
          // handle it
          if (retryPrimaryException(e)) {
            // restore updated versions...
            for (int j = 0; j < requestIndex; j++) {
              applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
            }
            throw (ElasticsearchException) e;
          }
          if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
            logger.trace(
                "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
          } else {
            logger.debug(
                "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
          }
          // if its a conflict failure, and we already executed the request on a primary (and we
          // execute it
          // again, due to primary relocation and only processing up to N bulk items when the shard
          // gets closed)
          // then just use the response we got from the successful execution
          if (item.getPrimaryResponse() != null && isConflictException(e)) {
            setResponse(item, item.getPrimaryResponse());
          } else {
            setResponse(
                item,
                new BulkItemResponse(
                    item.id(),
                    indexRequest.opType().lowercase(),
                    new BulkItemResponse.Failure(
                        request.index(), indexRequest.type(), indexRequest.id(), e)));
          }
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        preVersions[requestIndex] = deleteRequest.version();
        preVersionTypes[requestIndex] = deleteRequest.versionType();

        try {
          // add the response
          final WriteResult<DeleteResponse> writeResult =
              shardDeleteOperation(request, deleteRequest, indexShard);
          DeleteResponse deleteResponse = writeResult.response();
          location = locationToSync(location, writeResult.location);
          setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
        } catch (Throwable e) {
          // rethrow the failure if we are going to retry on primary and let parent failure to
          // handle it
          if (retryPrimaryException(e)) {
            // restore updated versions...
            for (int j = 0; j < requestIndex; j++) {
              applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
            }
            throw (ElasticsearchException) e;
          }
          if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
            logger.trace(
                "{} failed to execute bulk item (delete) {}",
                e,
                shardRequest.shardId,
                deleteRequest);
          } else {
            logger.debug(
                "{} failed to execute bulk item (delete) {}",
                e,
                shardRequest.shardId,
                deleteRequest);
          }
          // if its a conflict failure, and we already executed the request on a primary (and we
          // execute it
          // again, due to primary relocation and only processing up to N bulk items when the shard
          // gets closed)
          // then just use the response we got from the successful execution
          if (item.getPrimaryResponse() != null && isConflictException(e)) {
            setResponse(item, item.getPrimaryResponse());
          } else {
            setResponse(
                item,
                new BulkItemResponse(
                    item.id(),
                    OP_TYPE_DELETE,
                    new BulkItemResponse.Failure(
                        request.index(), deleteRequest.type(), deleteRequest.id(), e)));
          }
        }
      } else if (item.request() instanceof UpdateRequest) {
        UpdateRequest updateRequest = (UpdateRequest) item.request();
        preVersions[requestIndex] = updateRequest.version();
        preVersionTypes[requestIndex] = updateRequest.versionType();
        //  We need to do the requested retries plus the initial attempt. We don't do <
        // 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE
        for (int updateAttemptsCount = 0;
            updateAttemptsCount <= updateRequest.retryOnConflict();
            updateAttemptsCount++) {
          UpdateResult updateResult;
          try {
            updateResult = shardUpdateOperation(clusterState, request, updateRequest, indexShard);
          } catch (Throwable t) {
            updateResult = new UpdateResult(null, null, false, t, null);
          }
          if (updateResult.success()) {
            if (updateResult.writeResult != null) {
              location = locationToSync(location, updateResult.writeResult.location);
            }
            switch (updateResult.result.operation()) {
              case UPSERT:
              case INDEX:
                WriteResult<IndexResponse> result = updateResult.writeResult;
                IndexRequest indexRequest = updateResult.request();
                BytesReference indexSourceAsBytes = indexRequest.source();
                // add the response
                IndexResponse indexResponse = result.response();
                UpdateResponse updateResponse =
                    new UpdateResponse(
                        indexResponse.getShardInfo(),
                        indexResponse.getIndex(),
                        indexResponse.getType(),
                        indexResponse.getId(),
                        indexResponse.getVersion(),
                        indexResponse.isCreated());
                if (updateRequest.fields() != null && updateRequest.fields().length > 0) {
                  Tuple<XContentType, Map<String, Object>> sourceAndContent =
                      XContentHelper.convertToMap(indexSourceAsBytes, true);
                  updateResponse.setGetResult(
                      updateHelper.extractGetResult(
                          updateRequest,
                          shardRequest.request.index(),
                          indexResponse.getVersion(),
                          sourceAndContent.v2(),
                          sourceAndContent.v1(),
                          indexSourceAsBytes));
                }
                item =
                    request.items()[requestIndex] =
                        new BulkItemRequest(request.items()[requestIndex].id(), indexRequest);
                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
                break;
              case DELETE:
                WriteResult<DeleteResponse> writeResult = updateResult.writeResult;
                DeleteResponse response = writeResult.response();
                DeleteRequest deleteRequest = updateResult.request();
                updateResponse =
                    new UpdateResponse(
                        response.getShardInfo(),
                        response.getIndex(),
                        response.getType(),
                        response.getId(),
                        response.getVersion(),
                        false);
                updateResponse.setGetResult(
                    updateHelper.extractGetResult(
                        updateRequest,
                        shardRequest.request.index(),
                        response.getVersion(),
                        updateResult.result.updatedSourceAsMap(),
                        updateResult.result.updateSourceContentType(),
                        null));
                // Replace the update request to the translated delete request to execute on the
                // replica.
                item =
                    request.items()[requestIndex] =
                        new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest);
                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
                break;
              case NONE:
                setResponse(
                    item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult));
                item.setIgnoreOnReplica(); // no need to go to the replica
                break;
            }
            // NOTE: Breaking out of the retry_on_conflict loop!
            break;
          } else if (updateResult.failure()) {
            Throwable t = updateResult.error;
            if (updateResult.retry) {
              // updateAttemptCount is 0 based and marks current attempt, if it's equal to
              // retryOnConflict we are going out of the iteration
              if (updateAttemptsCount >= updateRequest.retryOnConflict()) {
                setResponse(
                    item,
                    new BulkItemResponse(
                        item.id(),
                        OP_TYPE_UPDATE,
                        new BulkItemResponse.Failure(
                            request.index(), updateRequest.type(), updateRequest.id(), t)));
              }
            } else {
              // rethrow the failure if we are going to retry on primary and let parent failure to
              // handle it
              if (retryPrimaryException(t)) {
                // restore updated versions...
                for (int j = 0; j < requestIndex; j++) {
                  applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
                }
                throw (ElasticsearchException) t;
              }
              // if its a conflict failure, and we already executed the request on a primary (and we
              // execute it
              // again, due to primary relocation and only processing up to N bulk items when the
              // shard gets closed)
              // then just use the response we got from the successful execution
              if (item.getPrimaryResponse() != null && isConflictException(t)) {
                setResponse(item, item.getPrimaryResponse());
              } else if (updateResult.result == null) {
                setResponse(
                    item,
                    new BulkItemResponse(
                        item.id(),
                        OP_TYPE_UPDATE,
                        new BulkItemResponse.Failure(
                            shardRequest.request.index(),
                            updateRequest.type(),
                            updateRequest.id(),
                            t)));
              } else {
                switch (updateResult.result.operation()) {
                  case UPSERT:
                  case INDEX:
                    IndexRequest indexRequest = updateResult.request();
                    if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
                      logger.trace(
                          "{} failed to execute bulk item (index) {}",
                          t,
                          shardRequest.shardId,
                          indexRequest);
                    } else {
                      logger.debug(
                          "{} failed to execute bulk item (index) {}",
                          t,
                          shardRequest.shardId,
                          indexRequest);
                    }
                    setResponse(
                        item,
                        new BulkItemResponse(
                            item.id(),
                            OP_TYPE_UPDATE,
                            new BulkItemResponse.Failure(
                                request.index(), indexRequest.type(), indexRequest.id(), t)));
                    break;
                  case DELETE:
                    DeleteRequest deleteRequest = updateResult.request();
                    if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
                      logger.trace(
                          "{} failed to execute bulk item (delete) {}",
                          t,
                          shardRequest.shardId,
                          deleteRequest);
                    } else {
                      logger.debug(
                          "{} failed to execute bulk item (delete) {}",
                          t,
                          shardRequest.shardId,
                          deleteRequest);
                    }
                    setResponse(
                        item,
                        new BulkItemResponse(
                            item.id(),
                            OP_TYPE_DELETE,
                            new BulkItemResponse.Failure(
                                request.index(), deleteRequest.type(), deleteRequest.id(), t)));
                    break;
                }
              }
              // NOTE: Breaking out of the retry_on_conflict loop!
              break;
            }
          }
        }
      } else {
        throw new IllegalStateException("Unexpected index operation: " + item.request());
      }

      assert item.getPrimaryResponse() != null;
      assert preVersionTypes[requestIndex] != null;
    }

    processAfter(request.refresh(), indexShard, location);
    BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
    BulkItemRequest[] items = request.items();
    for (int i = 0; i < items.length; i++) {
      responses[i] = items[i].getPrimaryResponse();
    }
    return new Tuple<>(
        new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request);
  }
コード例 #10
0
  private void executeBulk(
      final BulkRequest bulkRequest,
      final long startTime,
      final ActionListener<BulkResponse> listener,
      final AtomicArray<BulkItemResponse> responses) {
    final ClusterState clusterState = clusterService.state();
    // TODO use timeout to wait here if its blocked...
    clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);

    final ConcreteIndices concreteIndices =
        new ConcreteIndices(clusterState, indexNameExpressionResolver);
    MetaData metaData = clusterState.metaData();
    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof DocumentRequest) {
        DocumentRequest req = (DocumentRequest) request;

        if (addFailureIfIndexIsUnavailable(
            req, bulkRequest, responses, i, concreteIndices, metaData)) {
          continue;
        }

        String concreteIndex = concreteIndices.resolveIfAbsent(req);
        if (request instanceof IndexRequest) {
          IndexRequest indexRequest = (IndexRequest) request;
          MappingMetaData mappingMd = null;
          if (metaData.hasIndex(concreteIndex)) {
            mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type());
          }
          try {
            indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
          } catch (ElasticsearchParseException | RoutingMissingException e) {
            BulkItemResponse.Failure failure =
                new BulkItemResponse.Failure(
                    concreteIndex, indexRequest.type(), indexRequest.id(), e);
            BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
            responses.set(i, bulkItemResponse);
            // make sure the request gets never processed again
            bulkRequest.requests.set(i, null);
          }
        } else {
          concreteIndices.resolveIfAbsent(req);
          req.routing(
              clusterState
                  .metaData()
                  .resolveIndexRouting(req.parent(), req.routing(), req.index()));
        }
      }
    }

    // first, go over all the requests and create a ShardId -> Operations mapping
    Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();

    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index());
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    indexRequest.type(),
                    indexRequest.id(),
                    indexRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(deleteRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && deleteRequest.routing() == null) {
          // if routing is required, and no routing on the delete request, we need to broadcast
          // it....
          GroupShardsIterator groupShards =
              clusterService.operationRouting().broadcastDeleteShards(clusterState, concreteIndex);
          for (ShardIterator shardIt : groupShards) {
            List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
            if (list == null) {
              list = new ArrayList<>();
              requestsByShard.put(shardIt.shardId(), list);
            }
            list.add(new BulkItemRequest(i, deleteRequest));
          }
        } else {
          ShardId shardId =
              clusterService
                  .operationRouting()
                  .indexShards(
                      clusterState,
                      concreteIndex,
                      deleteRequest.type(),
                      deleteRequest.id(),
                      deleteRequest.routing())
                  .shardId();
          List<BulkItemRequest> list = requestsByShard.get(shardId);
          if (list == null) {
            list = new ArrayList<>();
            requestsByShard.put(shardId, list);
          }
          list.add(new BulkItemRequest(i, request));
        }
      } else if (request instanceof UpdateRequest) {
        UpdateRequest updateRequest = (UpdateRequest) request;
        String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index());
        MappingMetaData mappingMd =
            clusterState.metaData().index(concreteIndex).mappingOrDefault(updateRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && updateRequest.routing() == null) {
          BulkItemResponse.Failure failure =
              new BulkItemResponse.Failure(
                  updateRequest.index(),
                  updateRequest.type(),
                  updateRequest.id(),
                  new IllegalArgumentException("routing is required for this item"));
          responses.set(i, new BulkItemResponse(i, updateRequest.type(), failure));
          continue;
        }
        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    concreteIndex,
                    updateRequest.type(),
                    updateRequest.id(),
                    updateRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = new ArrayList<>();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      }
    }

    if (requestsByShard.isEmpty()) {
      listener.onResponse(
          new BulkResponse(
              responses.toArray(new BulkItemResponse[responses.length()]),
              buildTookInMillis(startTime)));
      return;
    }

    final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
    for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
      final ShardId shardId = entry.getKey();
      final List<BulkItemRequest> requests = entry.getValue();
      BulkShardRequest bulkShardRequest =
          new BulkShardRequest(
              bulkRequest,
              shardId,
              bulkRequest.refresh(),
              requests.toArray(new BulkItemRequest[requests.size()]));
      bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
      bulkShardRequest.timeout(bulkRequest.timeout());
      shardBulkAction.execute(
          bulkShardRequest,
          new ActionListener<BulkShardResponse>() {
            @Override
            public void onResponse(BulkShardResponse bulkShardResponse) {
              for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
                // we may have no response if item failed
                if (bulkItemResponse.getResponse() != null) {
                  bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
                }
                responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              for (BulkItemRequest request : requests) {
                if (request.request() instanceof IndexRequest) {
                  IndexRequest indexRequest = (IndexRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(indexRequest.index()),
                              indexRequest.type(),
                              indexRequest.id(),
                              e)));
                } else if (request.request() instanceof DeleteRequest) {
                  DeleteRequest deleteRequest = (DeleteRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "delete",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(deleteRequest.index()),
                              deleteRequest.type(),
                              deleteRequest.id(),
                              e)));
                } else if (request.request() instanceof UpdateRequest) {
                  UpdateRequest updateRequest = (UpdateRequest) request.request();
                  responses.set(
                      request.id(),
                      new BulkItemResponse(
                          request.id(),
                          "update",
                          new BulkItemResponse.Failure(
                              concreteIndices.getConcreteIndex(updateRequest.index()),
                              updateRequest.type(),
                              updateRequest.id(),
                              e)));
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(
                  new BulkResponse(
                      responses.toArray(new BulkItemResponse[responses.length()]),
                      buildTookInMillis(startTime)));
            }
          });
    }
  }
コード例 #11
0
  protected void shardOperation(
      final UpdateRequest request,
      final ActionListener<UpdateResponse> listener,
      final int retryCount)
      throws ElasticSearchException {
    IndexService indexService = indicesService.indexServiceSafe(request.index());
    IndexShard indexShard = indexService.shardSafe(request.shardId());

    long getDate = System.currentTimeMillis();
    final GetResult getResult =
        indexShard
            .getService()
            .get(
                request.type(),
                request.id(),
                new String[] {
                  SourceFieldMapper.NAME,
                  RoutingFieldMapper.NAME,
                  ParentFieldMapper.NAME,
                  TTLFieldMapper.NAME
                },
                true);

    // no doc, what to do, what to do...
    if (!getResult.isExists()) {
      if (request.upsertRequest() == null) {
        listener.onFailure(
            new DocumentMissingException(
                new ShardId(request.index(), request.shardId()), request.type(), request.id()));
        return;
      }
      final IndexRequest indexRequest = request.upsertRequest();
      indexRequest
          .index(request.index())
          .type(request.type())
          .id(request.id())
          // it has to be a "create!"
          .create(true)
          .routing(request.routing())
          .percolate(request.percolate())
          .refresh(request.refresh())
          .replicationType(request.replicationType())
          .consistencyLevel(request.consistencyLevel());
      indexRequest.operationThreaded(false);
      // we fetch it from the index request so we don't generate the bytes twice, its already done
      // in the index request
      final BytesReference updateSourceBytes = indexRequest.source();
      indexAction.execute(
          indexRequest,
          new ActionListener<IndexResponse>() {
            @Override
            public void onResponse(IndexResponse response) {
              UpdateResponse update =
                  new UpdateResponse(
                      response.getIndex(),
                      response.getType(),
                      response.getId(),
                      response.getVersion());
              update.setMatches(response.getMatches());
              if (request.fields() != null && request.fields().length > 0) {
                Tuple<XContentType, Map<String, Object>> sourceAndContent =
                    XContentHelper.convertToMap(updateSourceBytes, true);
                update.setGetResult(
                    extractGetResult(
                        request,
                        response.getVersion(),
                        sourceAndContent.v2(),
                        sourceAndContent.v1(),
                        updateSourceBytes));
              } else {
                update.setGetResult(null);
              }
              listener.onResponse(update);
            }

            @Override
            public void onFailure(Throwable e) {
              e = ExceptionsHelper.unwrapCause(e);
              if (e instanceof VersionConflictEngineException
                  || e instanceof DocumentAlreadyExistsException) {
                if (retryCount < request.retryOnConflict()) {
                  threadPool
                      .executor(executor())
                      .execute(
                          new Runnable() {
                            @Override
                            public void run() {
                              shardOperation(request, listener, retryCount + 1);
                            }
                          });
                  return;
                }
              }
              listener.onFailure(e);
            }
          });
      return;
    }

    if (getResult.internalSourceRef() == null) {
      // no source, we can't do nothing, through a failure...
      listener.onFailure(
          new DocumentSourceMissingException(
              new ShardId(request.index(), request.shardId()), request.type(), request.id()));
      return;
    }

    Tuple<XContentType, Map<String, Object>> sourceAndContent =
        XContentHelper.convertToMap(getResult.internalSourceRef(), true);
    String operation = null;
    String timestamp = null;
    Long ttl = null;
    Object fetchedTTL = null;
    final Map<String, Object> updatedSourceAsMap;
    final XContentType updateSourceContentType = sourceAndContent.v1();
    String routing =
        getResult.getFields().containsKey(RoutingFieldMapper.NAME)
            ? getResult.field(RoutingFieldMapper.NAME).getValue().toString()
            : null;
    String parent =
        getResult.getFields().containsKey(ParentFieldMapper.NAME)
            ? getResult.field(ParentFieldMapper.NAME).getValue().toString()
            : null;

    if (request.script() == null && request.doc() != null) {
      IndexRequest indexRequest = request.doc();
      updatedSourceAsMap = sourceAndContent.v2();
      if (indexRequest.ttl() > 0) {
        ttl = indexRequest.ttl();
      }
      timestamp = indexRequest.timestamp();
      if (indexRequest.routing() != null) {
        routing = indexRequest.routing();
      }
      if (indexRequest.parent() != null) {
        parent = indexRequest.parent();
      }
      XContentHelper.update(updatedSourceAsMap, indexRequest.sourceAsMap());
    } else {
      Map<String, Object> ctx = new HashMap<String, Object>(2);
      ctx.put("_source", sourceAndContent.v2());

      try {
        ExecutableScript script =
            scriptService.executable(request.scriptLang, request.script, request.scriptParams);
        script.setNextVar("ctx", ctx);
        script.run();
        // we need to unwrap the ctx...
        ctx = (Map<String, Object>) script.unwrap(ctx);
      } catch (Exception e) {
        throw new ElasticSearchIllegalArgumentException("failed to execute script", e);
      }

      operation = (String) ctx.get("op");
      timestamp = (String) ctx.get("_timestamp");
      fetchedTTL = ctx.get("_ttl");
      if (fetchedTTL != null) {
        if (fetchedTTL instanceof Number) {
          ttl = ((Number) fetchedTTL).longValue();
        } else {
          ttl = TimeValue.parseTimeValue((String) fetchedTTL, null).millis();
        }
      }

      updatedSourceAsMap = (Map<String, Object>) ctx.get("_source");
    }

    // apply script to update the source
    // No TTL has been given in the update script so we keep previous TTL value if there is one
    if (ttl == null) {
      ttl =
          getResult.getFields().containsKey(TTLFieldMapper.NAME)
              ? (Long) getResult.field(TTLFieldMapper.NAME).getValue()
              : null;
      if (ttl != null) {
        ttl =
            ttl
                - (System.currentTimeMillis()
                    - getDate); // It is an approximation of exact TTL value, could be improved
      }
    }

    // TODO: external version type, does it make sense here? does not seem like it...

    if (operation == null || "index".equals(operation)) {
      final IndexRequest indexRequest =
          Requests.indexRequest(request.index())
              .type(request.type())
              .id(request.id())
              .routing(routing)
              .parent(parent)
              .source(updatedSourceAsMap, updateSourceContentType)
              .version(getResult.getVersion())
              .replicationType(request.replicationType())
              .consistencyLevel(request.consistencyLevel())
              .timestamp(timestamp)
              .ttl(ttl)
              .percolate(request.percolate())
              .refresh(request.refresh());
      indexRequest.operationThreaded(false);
      // we fetch it from the index request so we don't generate the bytes twice, its already done
      // in the index request
      final BytesReference updateSourceBytes = indexRequest.source();
      indexAction.execute(
          indexRequest,
          new ActionListener<IndexResponse>() {
            @Override
            public void onResponse(IndexResponse response) {
              UpdateResponse update =
                  new UpdateResponse(
                      response.getIndex(),
                      response.getType(),
                      response.getId(),
                      response.getVersion());
              update.setMatches(response.getMatches());
              update.setGetResult(
                  extractGetResult(
                      request,
                      response.getVersion(),
                      updatedSourceAsMap,
                      updateSourceContentType,
                      updateSourceBytes));
              listener.onResponse(update);
            }

            @Override
            public void onFailure(Throwable e) {
              e = ExceptionsHelper.unwrapCause(e);
              if (e instanceof VersionConflictEngineException) {
                if (retryCount < request.retryOnConflict()) {
                  threadPool
                      .executor(executor())
                      .execute(
                          new Runnable() {
                            @Override
                            public void run() {
                              shardOperation(request, listener, retryCount + 1);
                            }
                          });
                  return;
                }
              }
              listener.onFailure(e);
            }
          });
    } else if ("delete".equals(operation)) {
      DeleteRequest deleteRequest =
          Requests.deleteRequest(request.index())
              .type(request.type())
              .id(request.id())
              .routing(routing)
              .parent(parent)
              .version(getResult.getVersion())
              .replicationType(request.replicationType())
              .consistencyLevel(request.consistencyLevel());
      deleteRequest.operationThreaded(false);
      deleteAction.execute(
          deleteRequest,
          new ActionListener<DeleteResponse>() {
            @Override
            public void onResponse(DeleteResponse response) {
              UpdateResponse update =
                  new UpdateResponse(
                      response.getIndex(),
                      response.getType(),
                      response.getId(),
                      response.getVersion());
              update.setGetResult(
                  extractGetResult(
                      request,
                      response.getVersion(),
                      updatedSourceAsMap,
                      updateSourceContentType,
                      null));
              listener.onResponse(update);
            }

            @Override
            public void onFailure(Throwable e) {
              e = ExceptionsHelper.unwrapCause(e);
              if (e instanceof VersionConflictEngineException) {
                if (retryCount < request.retryOnConflict()) {
                  threadPool
                      .executor(executor())
                      .execute(
                          new Runnable() {
                            @Override
                            public void run() {
                              shardOperation(request, listener, retryCount + 1);
                            }
                          });
                  return;
                }
              }
              listener.onFailure(e);
            }
          });
    } else if ("none".equals(operation)) {
      UpdateResponse update =
          new UpdateResponse(
              getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion());
      update.setGetResult(
          extractGetResult(
              request, getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, null));
      listener.onResponse(update);
    } else {
      logger.warn(
          "Used update operation [{}] for script [{}], doing nothing...",
          operation,
          request.script);
      listener.onResponse(
          new UpdateResponse(
              getResult.getIndex(),
              getResult.getType(),
              getResult.getId(),
              getResult.getVersion()));
    }
  }
コード例 #12
0
  private void executeBulk(
      final BulkRequest bulkRequest,
      final long startTime,
      final ActionListener<BulkResponse> listener) {
    ClusterState clusterState = clusterService.state();
    for (ActionRequest request : bulkRequest.requests) {
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        indexRequest.routing(
            clusterState
                .metaData()
                .resolveIndexRouting(indexRequest.routing(), indexRequest.index()));
        indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index()));
        if (allowIdGeneration) {
          if (indexRequest.id() == null) {
            indexRequest.id(UUID.randomBase64UUID());
            // since we generate the id, change it to CREATE
            indexRequest.opType(IndexRequest.OpType.CREATE);
          }
        }
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index()));
      }
    }
    final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()];

    // first, go over all the requests and create a ShardId -> Operations mapping
    Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap();
    for (int i = 0; i < bulkRequest.requests.size(); i++) {
      ActionRequest request = bulkRequest.requests.get(i);
      if (request instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) request;
        // handle routing
        MappingMetaData mappingMd =
            clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type());
        if (mappingMd != null) {
          try {
            indexRequest.processRouting(mappingMd);
          } catch (ElasticSearchException e) {
            responses[i] =
                new BulkItemResponse(
                    i,
                    indexRequest.opType().toString().toLowerCase(),
                    new BulkItemResponse.Failure(
                        indexRequest.index(),
                        indexRequest.type(),
                        indexRequest.id(),
                        e.getDetailedMessage()));
            continue;
          }
        }

        ShardId shardId =
            clusterService
                .operationRouting()
                .indexShards(
                    clusterState,
                    indexRequest.index(),
                    indexRequest.type(),
                    indexRequest.id(),
                    indexRequest.routing())
                .shardId();
        List<BulkItemRequest> list = requestsByShard.get(shardId);
        if (list == null) {
          list = Lists.newArrayList();
          requestsByShard.put(shardId, list);
        }
        list.add(new BulkItemRequest(i, request));
      } else if (request instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) request;
        MappingMetaData mappingMd =
            clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type());
        if (mappingMd != null
            && mappingMd.routing().required()
            && deleteRequest.routing() == null) {
          // if routing is required, and no routing on the delete request, we need to broadcast
          // it....
          GroupShardsIterator groupShards =
              clusterService
                  .operationRouting()
                  .broadcastDeleteShards(clusterState, deleteRequest.index());
          for (ShardIterator shardIt : groupShards) {
            List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
            if (list == null) {
              list = Lists.newArrayList();
              requestsByShard.put(shardIt.shardId(), list);
            }
            list.add(new BulkItemRequest(i, request));
          }
        } else {
          ShardId shardId =
              clusterService
                  .operationRouting()
                  .deleteShards(
                      clusterState,
                      deleteRequest.index(),
                      deleteRequest.type(),
                      deleteRequest.id(),
                      deleteRequest.routing())
                  .shardId();
          List<BulkItemRequest> list = requestsByShard.get(shardId);
          if (list == null) {
            list = Lists.newArrayList();
            requestsByShard.put(shardId, list);
          }
          list.add(new BulkItemRequest(i, request));
        }
      }
    }

    if (requestsByShard.isEmpty()) {
      listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime));
      return;
    }

    final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
    for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
      final ShardId shardId = entry.getKey();
      final List<BulkItemRequest> requests = entry.getValue();
      BulkShardRequest bulkShardRequest =
          new BulkShardRequest(
              shardId.index().name(),
              shardId.id(),
              bulkRequest.refresh(),
              requests.toArray(new BulkItemRequest[requests.size()]));
      bulkShardRequest.replicationType(bulkRequest.replicationType());
      bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
      shardBulkAction.execute(
          bulkShardRequest,
          new ActionListener<BulkShardResponse>() {
            @Override
            public void onResponse(BulkShardResponse bulkShardResponse) {
              synchronized (responses) {
                for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) {
                  responses[bulkItemResponse.itemId()] = bulkItemResponse;
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            @Override
            public void onFailure(Throwable e) {
              // create failures for all relevant requests
              String message = ExceptionsHelper.detailedMessage(e);
              synchronized (responses) {
                for (BulkItemRequest request : requests) {
                  if (request.request() instanceof IndexRequest) {
                    IndexRequest indexRequest = (IndexRequest) request.request();
                    responses[request.id()] =
                        new BulkItemResponse(
                            request.id(),
                            indexRequest.opType().toString().toLowerCase(),
                            new BulkItemResponse.Failure(
                                indexRequest.index(),
                                indexRequest.type(),
                                indexRequest.id(),
                                message));
                  } else if (request.request() instanceof DeleteRequest) {
                    DeleteRequest deleteRequest = (DeleteRequest) request.request();
                    responses[request.id()] =
                        new BulkItemResponse(
                            request.id(),
                            "delete",
                            new BulkItemResponse.Failure(
                                deleteRequest.index(),
                                deleteRequest.type(),
                                deleteRequest.id(),
                                message));
                  }
                }
              }
              if (counter.decrementAndGet() == 0) {
                finishHim();
              }
            }

            private void finishHim() {
              listener.onResponse(
                  new BulkResponse(responses, System.currentTimeMillis() - startTime));
            }
          });
    }
  }