private WriteResult<DeleteResponse> shardDeleteOperation(
      BulkShardRequest request, DeleteRequest deleteRequest, IndexShard indexShard) {
    Engine.Delete delete =
        indexShard.prepareDelete(
            deleteRequest.type(),
            deleteRequest.id(),
            deleteRequest.version(),
            deleteRequest.versionType(),
            Engine.Operation.Origin.PRIMARY);
    indexShard.delete(delete);
    // update the request with the version so it will go to the replicas
    deleteRequest.versionType(delete.versionType().versionTypeForReplicationAndRecovery());
    deleteRequest.version(delete.version());

    assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version());

    DeleteResponse deleteResponse =
        new DeleteResponse(
            request.index(),
            deleteRequest.type(),
            deleteRequest.id(),
            delete.version(),
            delete.found());
    return new WriteResult(deleteResponse, delete.getTranslogLocation());
  }
Пример #2
0
 public DeleteRequest(DeleteRequest request) {
   super(request);
   this.type = request.type();
   this.id = request.id();
   this.routing = request.routing();
   this.refresh = request.refresh();
   this.version = request.version();
   this.versionType = request.versionType();
 }
Пример #3
0
  @Override
  public void handleRequest(
      final RestRequest request, final RestChannel channel, final NodeClient client) {
    DeleteRequest deleteRequest =
        new DeleteRequest(request.param("index"), request.param("type"), request.param("id"));
    deleteRequest.routing(request.param("routing"));
    deleteRequest.parent(
        request.param(
            "parent")); // order is important, set it after routing, so it will set the routing
    deleteRequest.timeout(request.paramAsTime("timeout", DeleteRequest.DEFAULT_TIMEOUT));
    deleteRequest.setRefreshPolicy(request.param("refresh"));
    deleteRequest.version(RestActions.parseVersion(request));
    deleteRequest.versionType(
        VersionType.fromString(request.param("version_type"), deleteRequest.versionType()));

    String consistencyLevel = request.param("consistency");
    if (consistencyLevel != null) {
      deleteRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel));
    }

    client.delete(deleteRequest, new RestStatusToXContentListener<>(channel));
  }
 @Override
 protected PrimaryResponse<IngestShardResponse, IngestShardRequest> shardOperationOnPrimary(
     ClusterState clusterState, PrimaryOperationRequest shardRequest) {
   final IngestShardRequest request = shardRequest.request;
   IndexShard indexShard =
       indicesService
           .indexServiceSafe(shardRequest.request.index())
           .shardSafe(shardRequest.shardId);
   int successSize = 0;
   List<IngestItemFailure> failure = newLinkedList();
   int size = request.items().size();
   long[] versions = new long[size];
   Set<Tuple<String, String>> mappingsToUpdate = newHashSet();
   for (int i = 0; i < size; i++) {
     IngestItemRequest item = request.items().get(i);
     if (item.request() instanceof IndexRequest) {
       IndexRequest indexRequest = (IndexRequest) item.request();
       Engine.IndexingOperation op = null;
       try {
         // validate, if routing is required, that we got routing
         MappingMetaData mappingMd =
             clusterState.metaData().index(request.index()).mappingOrDefault(indexRequest.type());
         if (mappingMd != null && mappingMd.routing().required()) {
           if (indexRequest.routing() == null) {
             throw new RoutingMissingException(
                 indexRequest.index(), indexRequest.type(), indexRequest.id());
           }
         }
         SourceToParse sourceToParse =
             SourceToParse.source(SourceToParse.Origin.PRIMARY, indexRequest.source())
                 .type(indexRequest.type())
                 .id(indexRequest.id())
                 .routing(indexRequest.routing())
                 .parent(indexRequest.parent())
                 .timestamp(indexRequest.timestamp())
                 .ttl(indexRequest.ttl());
         long version;
         if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
           Engine.Index index =
               indexShard
                   .prepareIndex(sourceToParse)
                   .version(indexRequest.version())
                   .versionType(indexRequest.versionType())
                   .origin(Engine.Operation.Origin.PRIMARY);
           op = index;
           indexShard.index(index);
           version = index.version();
         } else {
           Engine.Create create =
               indexShard
                   .prepareCreate(sourceToParse)
                   .version(indexRequest.version())
                   .versionType(indexRequest.versionType())
                   .origin(Engine.Operation.Origin.PRIMARY);
           op = create;
           indexShard.create(create);
           version = create.version();
         }
         versions[i] = indexRequest.version();
         // update the version on request so it will happen on the replicas
         indexRequest.version(version);
         successSize++;
       } catch (Throwable e) {
         // rethrow the failure if we are going to retry on primary and let parent failure to
         // handle it
         if (retryPrimaryException(e)) {
           // restore updated versions...
           for (int j = 0; j < i; j++) {
             applyVersion(request.items().get(j), versions[j]);
           }
           logger.error(e.getMessage(), e);
           throw new ElasticsearchException(e.getMessage());
         }
         if (e instanceof ElasticsearchException
             && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
           logger.error(
               "[{}][{}] failed to execute bulk item (index) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               indexRequest);
         } else {
           logger.error(
               "[{}][{}] failed to execute bulk item (index) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               indexRequest);
         }
         failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
         // nullify the request so it won't execute on the replicas
         request.items().set(i, null);
       } finally {
         // update mapping on master if needed, we won't update changes to the same type, since
         // once its changed, it won't have mappers added
         if (op != null && op.parsedDoc().mappingsModified()) {
           mappingsToUpdate.add(Tuple.tuple(indexRequest.index(), indexRequest.type()));
         }
       }
     } else if (item.request() instanceof DeleteRequest) {
       DeleteRequest deleteRequest = (DeleteRequest) item.request();
       try {
         Engine.Delete delete =
             indexShard
                 .prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
                 .versionType(deleteRequest.versionType())
                 .origin(Engine.Operation.Origin.PRIMARY);
         indexShard.delete(delete);
         // update the request with teh version so it will go to the replicas
         deleteRequest.version(delete.version());
         successSize++;
       } catch (Throwable e) {
         // rethrow the failure if we are going to retry on primary and let parent failure to
         // handle it
         if (retryPrimaryException(e)) {
           // restore updated versions...
           for (int j = 0; j < i; j++) {
             applyVersion(request.items().get(j), versions[j]);
           }
           logger.error(e.getMessage(), e);
           throw new ElasticsearchException(e.getMessage());
         }
         if (e instanceof ElasticsearchException
             && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
           logger.trace(
               "[{}][{}] failed to execute bulk item (delete) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               deleteRequest);
         } else {
           logger.debug(
               "[{}][{}] failed to execute bulk item (delete) {}",
               e,
               shardRequest.request.index(),
               shardRequest.shardId,
               deleteRequest);
         }
         failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
         // nullify the request so it won't execute on the replicas
         request.items().set(i, null);
       }
     }
   }
   if (!mappingsToUpdate.isEmpty()) {
     for (Tuple<String, String> mappingToUpdate : mappingsToUpdate) {
       logger.info("mapping update {} {}", mappingToUpdate.v1(), mappingToUpdate.v2());
       updateMappingOnMaster(mappingToUpdate.v1(), mappingToUpdate.v2());
     }
   }
   IngestShardResponse response =
       new IngestShardResponse(
           new ShardId(request.index(), request.shardId()), successSize, failure);
   return new PrimaryResponse<IngestShardResponse, IngestShardRequest>(
       shardRequest.request, response, null);
 }
  @Override
  protected void shardOperationOnReplica(ShardId shardId, BulkShardRequest request) {
    IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
    IndexShard indexShard = indexService.shardSafe(shardId.id());
    Translog.Location location = null;
    for (int i = 0; i < request.items().length; i++) {
      BulkItemRequest item = request.items()[i];
      if (item == null || item.isIgnoreOnReplica()) {
        continue;
      }
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        try {
          SourceToParse sourceToParse =
              SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
                  .index(shardId.getIndex())
                  .type(indexRequest.type())
                  .id(indexRequest.id())
                  .routing(indexRequest.routing())
                  .parent(indexRequest.parent())
                  .timestamp(indexRequest.timestamp())
                  .ttl(indexRequest.ttl());

          final Engine.IndexingOperation operation;
          if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
            operation =
                indexShard.prepareIndex(
                    sourceToParse,
                    indexRequest.version(),
                    indexRequest.versionType(),
                    Engine.Operation.Origin.REPLICA,
                    request.canHaveDuplicates() || indexRequest.canHaveDuplicates());
          } else {
            assert indexRequest.opType() == IndexRequest.OpType.CREATE : indexRequest.opType();
            operation =
                indexShard.prepareCreate(
                    sourceToParse,
                    indexRequest.version(),
                    indexRequest.versionType(),
                    Engine.Operation.Origin.REPLICA,
                    request.canHaveDuplicates() || indexRequest.canHaveDuplicates(),
                    indexRequest.autoGeneratedId());
          }
          Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
          if (update != null) {
            throw new RetryOnReplicaException(
                shardId,
                "Mappings are not available on the replica yet, triggered update: " + update);
          }
          operation.execute(indexShard);
          location = locationToSync(location, operation.getTranslogLocation());
        } catch (Throwable e) {
          // if its not an ignore replica failure, we need to make sure to bubble up the failure
          // so we will fail the shard
          if (!ignoreReplicaException(e)) {
            throw e;
          }
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        try {
          Engine.Delete delete =
              indexShard.prepareDelete(
                  deleteRequest.type(),
                  deleteRequest.id(),
                  deleteRequest.version(),
                  deleteRequest.versionType(),
                  Engine.Operation.Origin.REPLICA);
          indexShard.delete(delete);
          location = locationToSync(location, delete.getTranslogLocation());
        } catch (Throwable e) {
          // if its not an ignore replica failure, we need to make sure to bubble up the failure
          // so we will fail the shard
          if (!ignoreReplicaException(e)) {
            throw e;
          }
        }
      } else {
        throw new IllegalStateException("Unexpected index operation: " + item.request());
      }
    }

    processAfter(request.refresh(), indexShard, location);
  }
  @Override
  protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(
      ClusterState clusterState, PrimaryOperationRequest shardRequest) {
    final BulkShardRequest request = shardRequest.request;
    final IndexService indexService = indicesService.indexServiceSafe(request.index());
    final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());

    long[] preVersions = new long[request.items().length];
    VersionType[] preVersionTypes = new VersionType[request.items().length];
    Translog.Location location = null;
    for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
      BulkItemRequest item = request.items()[requestIndex];
      if (item.request() instanceof IndexRequest) {
        IndexRequest indexRequest = (IndexRequest) item.request();
        preVersions[requestIndex] = indexRequest.version();
        preVersionTypes[requestIndex] = indexRequest.versionType();
        try {
          WriteResult<IndexResponse> result =
              shardIndexOperation(request, indexRequest, clusterState, indexShard, true);
          location = locationToSync(location, result.location);
          // add the response
          IndexResponse indexResponse = result.response();
          setResponse(
              item,
              new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
        } catch (Throwable e) {
          // rethrow the failure if we are going to retry on primary and let parent failure to
          // handle it
          if (retryPrimaryException(e)) {
            // restore updated versions...
            for (int j = 0; j < requestIndex; j++) {
              applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
            }
            throw (ElasticsearchException) e;
          }
          if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
            logger.trace(
                "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
          } else {
            logger.debug(
                "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
          }
          // if its a conflict failure, and we already executed the request on a primary (and we
          // execute it
          // again, due to primary relocation and only processing up to N bulk items when the shard
          // gets closed)
          // then just use the response we got from the successful execution
          if (item.getPrimaryResponse() != null && isConflictException(e)) {
            setResponse(item, item.getPrimaryResponse());
          } else {
            setResponse(
                item,
                new BulkItemResponse(
                    item.id(),
                    indexRequest.opType().lowercase(),
                    new BulkItemResponse.Failure(
                        request.index(), indexRequest.type(), indexRequest.id(), e)));
          }
        }
      } else if (item.request() instanceof DeleteRequest) {
        DeleteRequest deleteRequest = (DeleteRequest) item.request();
        preVersions[requestIndex] = deleteRequest.version();
        preVersionTypes[requestIndex] = deleteRequest.versionType();

        try {
          // add the response
          final WriteResult<DeleteResponse> writeResult =
              shardDeleteOperation(request, deleteRequest, indexShard);
          DeleteResponse deleteResponse = writeResult.response();
          location = locationToSync(location, writeResult.location);
          setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
        } catch (Throwable e) {
          // rethrow the failure if we are going to retry on primary and let parent failure to
          // handle it
          if (retryPrimaryException(e)) {
            // restore updated versions...
            for (int j = 0; j < requestIndex; j++) {
              applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
            }
            throw (ElasticsearchException) e;
          }
          if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
            logger.trace(
                "{} failed to execute bulk item (delete) {}",
                e,
                shardRequest.shardId,
                deleteRequest);
          } else {
            logger.debug(
                "{} failed to execute bulk item (delete) {}",
                e,
                shardRequest.shardId,
                deleteRequest);
          }
          // if its a conflict failure, and we already executed the request on a primary (and we
          // execute it
          // again, due to primary relocation and only processing up to N bulk items when the shard
          // gets closed)
          // then just use the response we got from the successful execution
          if (item.getPrimaryResponse() != null && isConflictException(e)) {
            setResponse(item, item.getPrimaryResponse());
          } else {
            setResponse(
                item,
                new BulkItemResponse(
                    item.id(),
                    OP_TYPE_DELETE,
                    new BulkItemResponse.Failure(
                        request.index(), deleteRequest.type(), deleteRequest.id(), e)));
          }
        }
      } else if (item.request() instanceof UpdateRequest) {
        UpdateRequest updateRequest = (UpdateRequest) item.request();
        preVersions[requestIndex] = updateRequest.version();
        preVersionTypes[requestIndex] = updateRequest.versionType();
        //  We need to do the requested retries plus the initial attempt. We don't do <
        // 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE
        for (int updateAttemptsCount = 0;
            updateAttemptsCount <= updateRequest.retryOnConflict();
            updateAttemptsCount++) {
          UpdateResult updateResult;
          try {
            updateResult = shardUpdateOperation(clusterState, request, updateRequest, indexShard);
          } catch (Throwable t) {
            updateResult = new UpdateResult(null, null, false, t, null);
          }
          if (updateResult.success()) {
            if (updateResult.writeResult != null) {
              location = locationToSync(location, updateResult.writeResult.location);
            }
            switch (updateResult.result.operation()) {
              case UPSERT:
              case INDEX:
                WriteResult<IndexResponse> result = updateResult.writeResult;
                IndexRequest indexRequest = updateResult.request();
                BytesReference indexSourceAsBytes = indexRequest.source();
                // add the response
                IndexResponse indexResponse = result.response();
                UpdateResponse updateResponse =
                    new UpdateResponse(
                        indexResponse.getShardInfo(),
                        indexResponse.getIndex(),
                        indexResponse.getType(),
                        indexResponse.getId(),
                        indexResponse.getVersion(),
                        indexResponse.isCreated());
                if (updateRequest.fields() != null && updateRequest.fields().length > 0) {
                  Tuple<XContentType, Map<String, Object>> sourceAndContent =
                      XContentHelper.convertToMap(indexSourceAsBytes, true);
                  updateResponse.setGetResult(
                      updateHelper.extractGetResult(
                          updateRequest,
                          shardRequest.request.index(),
                          indexResponse.getVersion(),
                          sourceAndContent.v2(),
                          sourceAndContent.v1(),
                          indexSourceAsBytes));
                }
                item =
                    request.items()[requestIndex] =
                        new BulkItemRequest(request.items()[requestIndex].id(), indexRequest);
                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
                break;
              case DELETE:
                WriteResult<DeleteResponse> writeResult = updateResult.writeResult;
                DeleteResponse response = writeResult.response();
                DeleteRequest deleteRequest = updateResult.request();
                updateResponse =
                    new UpdateResponse(
                        response.getShardInfo(),
                        response.getIndex(),
                        response.getType(),
                        response.getId(),
                        response.getVersion(),
                        false);
                updateResponse.setGetResult(
                    updateHelper.extractGetResult(
                        updateRequest,
                        shardRequest.request.index(),
                        response.getVersion(),
                        updateResult.result.updatedSourceAsMap(),
                        updateResult.result.updateSourceContentType(),
                        null));
                // Replace the update request to the translated delete request to execute on the
                // replica.
                item =
                    request.items()[requestIndex] =
                        new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest);
                setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
                break;
              case NONE:
                setResponse(
                    item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult));
                item.setIgnoreOnReplica(); // no need to go to the replica
                break;
            }
            // NOTE: Breaking out of the retry_on_conflict loop!
            break;
          } else if (updateResult.failure()) {
            Throwable t = updateResult.error;
            if (updateResult.retry) {
              // updateAttemptCount is 0 based and marks current attempt, if it's equal to
              // retryOnConflict we are going out of the iteration
              if (updateAttemptsCount >= updateRequest.retryOnConflict()) {
                setResponse(
                    item,
                    new BulkItemResponse(
                        item.id(),
                        OP_TYPE_UPDATE,
                        new BulkItemResponse.Failure(
                            request.index(), updateRequest.type(), updateRequest.id(), t)));
              }
            } else {
              // rethrow the failure if we are going to retry on primary and let parent failure to
              // handle it
              if (retryPrimaryException(t)) {
                // restore updated versions...
                for (int j = 0; j < requestIndex; j++) {
                  applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
                }
                throw (ElasticsearchException) t;
              }
              // if its a conflict failure, and we already executed the request on a primary (and we
              // execute it
              // again, due to primary relocation and only processing up to N bulk items when the
              // shard gets closed)
              // then just use the response we got from the successful execution
              if (item.getPrimaryResponse() != null && isConflictException(t)) {
                setResponse(item, item.getPrimaryResponse());
              } else if (updateResult.result == null) {
                setResponse(
                    item,
                    new BulkItemResponse(
                        item.id(),
                        OP_TYPE_UPDATE,
                        new BulkItemResponse.Failure(
                            shardRequest.request.index(),
                            updateRequest.type(),
                            updateRequest.id(),
                            t)));
              } else {
                switch (updateResult.result.operation()) {
                  case UPSERT:
                  case INDEX:
                    IndexRequest indexRequest = updateResult.request();
                    if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
                      logger.trace(
                          "{} failed to execute bulk item (index) {}",
                          t,
                          shardRequest.shardId,
                          indexRequest);
                    } else {
                      logger.debug(
                          "{} failed to execute bulk item (index) {}",
                          t,
                          shardRequest.shardId,
                          indexRequest);
                    }
                    setResponse(
                        item,
                        new BulkItemResponse(
                            item.id(),
                            OP_TYPE_UPDATE,
                            new BulkItemResponse.Failure(
                                request.index(), indexRequest.type(), indexRequest.id(), t)));
                    break;
                  case DELETE:
                    DeleteRequest deleteRequest = updateResult.request();
                    if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
                      logger.trace(
                          "{} failed to execute bulk item (delete) {}",
                          t,
                          shardRequest.shardId,
                          deleteRequest);
                    } else {
                      logger.debug(
                          "{} failed to execute bulk item (delete) {}",
                          t,
                          shardRequest.shardId,
                          deleteRequest);
                    }
                    setResponse(
                        item,
                        new BulkItemResponse(
                            item.id(),
                            OP_TYPE_DELETE,
                            new BulkItemResponse.Failure(
                                request.index(), deleteRequest.type(), deleteRequest.id(), t)));
                    break;
                }
              }
              // NOTE: Breaking out of the retry_on_conflict loop!
              break;
            }
          }
        }
      } else {
        throw new IllegalStateException("Unexpected index operation: " + item.request());
      }

      assert item.getPrimaryResponse() != null;
      assert preVersionTypes[requestIndex] != null;
    }

    processAfter(request.refresh(), indexShard, location);
    BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
    BulkItemRequest[] items = request.items();
    for (int i = 0; i < items.length; i++) {
      responses[i] = items[i].getPrimaryResponse();
    }
    return new Tuple<>(
        new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request);
  }