private WriteResult<DeleteResponse> shardDeleteOperation( BulkShardRequest request, DeleteRequest deleteRequest, IndexShard indexShard) { Engine.Delete delete = indexShard.prepareDelete( deleteRequest.type(), deleteRequest.id(), deleteRequest.version(), deleteRequest.versionType(), Engine.Operation.Origin.PRIMARY); indexShard.delete(delete); // update the request with the version so it will go to the replicas deleteRequest.versionType(delete.versionType().versionTypeForReplicationAndRecovery()); deleteRequest.version(delete.version()); assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version()); DeleteResponse deleteResponse = new DeleteResponse( request.index(), deleteRequest.type(), deleteRequest.id(), delete.version(), delete.found()); return new WriteResult(deleteResponse, delete.getTranslogLocation()); }
public DeleteRequest(DeleteRequest request) { super(request); this.type = request.type(); this.id = request.id(); this.routing = request.routing(); this.refresh = request.refresh(); this.version = request.version(); this.versionType = request.versionType(); }
@Override protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) { IndexShard indexShard = indicesService .indexServiceSafe(shardRequest.request.index()) .shardSafe(shardRequest.shardId); final IngestShardRequest request = shardRequest.request; int size = request.items().size(); for (int i = 0; i < size; i++) { IngestItemRequest item = request.items().get(i); if (item == null) { continue; } if (item.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) item.request(); try { SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source()) .type(indexRequest.type()) .id(indexRequest.id()) .routing(indexRequest.routing()) .parent(indexRequest.parent()) .timestamp(indexRequest.timestamp()) .ttl(indexRequest.ttl()); if (indexRequest.opType() == IndexRequest.OpType.INDEX) { Engine.Index index = indexShard .prepareIndex(sourceToParse) .version(indexRequest.version()) .origin(Engine.Operation.Origin.REPLICA); indexShard.index(index); } else { Engine.Create create = indexShard .prepareCreate(sourceToParse) .version(indexRequest.version()) .origin(Engine.Operation.Origin.REPLICA); indexShard.create(create); } } catch (Exception e) { // ignore, we are on backup } } else if (item.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) item.request(); try { Engine.Delete delete = indexShard .prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version()) .origin(Engine.Operation.Origin.REPLICA); indexShard.delete(delete); } catch (Exception e) { // ignore, we are on backup } } } }
private boolean setResponseFailureIfIndexMatches( AtomicArray<BulkItemResponse> responses, int idx, ActionRequest request, String index, Throwable e) { if (request instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request; if (index.equals(indexRequest.index())) { responses.set( idx, new BulkItemResponse( idx, "index", new BulkItemResponse.Failure( indexRequest.index(), indexRequest.type(), indexRequest.id(), e))); return true; } } else if (request instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request; if (index.equals(deleteRequest.index())) { responses.set( idx, new BulkItemResponse( idx, "delete", new BulkItemResponse.Failure( deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), e))); return true; } } else if (request instanceof UpdateRequest) { UpdateRequest updateRequest = (UpdateRequest) request; if (index.equals(updateRequest.index())) { responses.set( idx, new BulkItemResponse( idx, "update", new BulkItemResponse.Failure( updateRequest.index(), updateRequest.type(), updateRequest.id(), e))); return true; } } else { throw new ElasticsearchException( "Parsed unknown request in bulk actions: " + request.getClass().getSimpleName()); } return false; }
@Override protected PrimaryResponse<IngestShardResponse, IngestShardRequest> shardOperationOnPrimary( ClusterState clusterState, PrimaryOperationRequest shardRequest) { final IngestShardRequest request = shardRequest.request; IndexShard indexShard = indicesService .indexServiceSafe(shardRequest.request.index()) .shardSafe(shardRequest.shardId); int successSize = 0; List<IngestItemFailure> failure = newLinkedList(); int size = request.items().size(); long[] versions = new long[size]; Set<Tuple<String, String>> mappingsToUpdate = newHashSet(); for (int i = 0; i < size; i++) { IngestItemRequest item = request.items().get(i); if (item.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) item.request(); Engine.IndexingOperation op = null; try { // validate, if routing is required, that we got routing MappingMetaData mappingMd = clusterState.metaData().index(request.index()).mappingOrDefault(indexRequest.type()); if (mappingMd != null && mappingMd.routing().required()) { if (indexRequest.routing() == null) { throw new RoutingMissingException( indexRequest.index(), indexRequest.type(), indexRequest.id()); } } SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, indexRequest.source()) .type(indexRequest.type()) .id(indexRequest.id()) .routing(indexRequest.routing()) .parent(indexRequest.parent()) .timestamp(indexRequest.timestamp()) .ttl(indexRequest.ttl()); long version; if (indexRequest.opType() == IndexRequest.OpType.INDEX) { Engine.Index index = indexShard .prepareIndex(sourceToParse) .version(indexRequest.version()) .versionType(indexRequest.versionType()) .origin(Engine.Operation.Origin.PRIMARY); op = index; indexShard.index(index); version = index.version(); } else { Engine.Create create = indexShard .prepareCreate(sourceToParse) .version(indexRequest.version()) .versionType(indexRequest.versionType()) .origin(Engine.Operation.Origin.PRIMARY); op = create; indexShard.create(create); version = create.version(); } versions[i] = indexRequest.version(); // update the version on request so it will happen on the replicas indexRequest.version(version); successSize++; } catch (Throwable e) { // rethrow the failure if we are going to retry on primary and let parent failure to // handle it if (retryPrimaryException(e)) { // restore updated versions... for (int j = 0; j < i; j++) { applyVersion(request.items().get(j), versions[j]); } logger.error(e.getMessage(), e); throw new ElasticsearchException(e.getMessage()); } if (e instanceof ElasticsearchException && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) { logger.error( "[{}][{}] failed to execute bulk item (index) {}", e, shardRequest.request.index(), shardRequest.shardId, indexRequest); } else { logger.error( "[{}][{}] failed to execute bulk item (index) {}", e, shardRequest.request.index(), shardRequest.shardId, indexRequest); } failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e))); // nullify the request so it won't execute on the replicas request.items().set(i, null); } finally { // update mapping on master if needed, we won't update changes to the same type, since // once its changed, it won't have mappers added if (op != null && op.parsedDoc().mappingsModified()) { mappingsToUpdate.add(Tuple.tuple(indexRequest.index(), indexRequest.type())); } } } else if (item.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) item.request(); try { Engine.Delete delete = indexShard .prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version()) .versionType(deleteRequest.versionType()) .origin(Engine.Operation.Origin.PRIMARY); indexShard.delete(delete); // update the request with teh version so it will go to the replicas deleteRequest.version(delete.version()); successSize++; } catch (Throwable e) { // rethrow the failure if we are going to retry on primary and let parent failure to // handle it if (retryPrimaryException(e)) { // restore updated versions... for (int j = 0; j < i; j++) { applyVersion(request.items().get(j), versions[j]); } logger.error(e.getMessage(), e); throw new ElasticsearchException(e.getMessage()); } if (e instanceof ElasticsearchException && ((ElasticsearchException) e).status() == RestStatus.CONFLICT) { logger.trace( "[{}][{}] failed to execute bulk item (delete) {}", e, shardRequest.request.index(), shardRequest.shardId, deleteRequest); } else { logger.debug( "[{}][{}] failed to execute bulk item (delete) {}", e, shardRequest.request.index(), shardRequest.shardId, deleteRequest); } failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e))); // nullify the request so it won't execute on the replicas request.items().set(i, null); } } } if (!mappingsToUpdate.isEmpty()) { for (Tuple<String, String> mappingToUpdate : mappingsToUpdate) { logger.info("mapping update {} {}", mappingToUpdate.v1(), mappingToUpdate.v2()); updateMappingOnMaster(mappingToUpdate.v1(), mappingToUpdate.v2()); } } IngestShardResponse response = new IngestShardResponse( new ShardId(request.index(), request.shardId()), successSize, failure); return new PrimaryResponse<IngestShardResponse, IngestShardRequest>( shardRequest.request, response, null); }
@Override protected void shardOperationOnReplica(ShardId shardId, BulkShardRequest request) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.shardSafe(shardId.id()); Translog.Location location = null; for (int i = 0; i < request.items().length; i++) { BulkItemRequest item = request.items()[i]; if (item == null || item.isIgnoreOnReplica()) { continue; } if (item.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) item.request(); try { SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source()) .index(shardId.getIndex()) .type(indexRequest.type()) .id(indexRequest.id()) .routing(indexRequest.routing()) .parent(indexRequest.parent()) .timestamp(indexRequest.timestamp()) .ttl(indexRequest.ttl()); final Engine.IndexingOperation operation; if (indexRequest.opType() == IndexRequest.OpType.INDEX) { operation = indexShard.prepareIndex( sourceToParse, indexRequest.version(), indexRequest.versionType(), Engine.Operation.Origin.REPLICA, request.canHaveDuplicates() || indexRequest.canHaveDuplicates()); } else { assert indexRequest.opType() == IndexRequest.OpType.CREATE : indexRequest.opType(); operation = indexShard.prepareCreate( sourceToParse, indexRequest.version(), indexRequest.versionType(), Engine.Operation.Origin.REPLICA, request.canHaveDuplicates() || indexRequest.canHaveDuplicates(), indexRequest.autoGeneratedId()); } Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { throw new RetryOnReplicaException( shardId, "Mappings are not available on the replica yet, triggered update: " + update); } operation.execute(indexShard); location = locationToSync(location, operation.getTranslogLocation()); } catch (Throwable e) { // if its not an ignore replica failure, we need to make sure to bubble up the failure // so we will fail the shard if (!ignoreReplicaException(e)) { throw e; } } } else if (item.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) item.request(); try { Engine.Delete delete = indexShard.prepareDelete( deleteRequest.type(), deleteRequest.id(), deleteRequest.version(), deleteRequest.versionType(), Engine.Operation.Origin.REPLICA); indexShard.delete(delete); location = locationToSync(location, delete.getTranslogLocation()); } catch (Throwable e) { // if its not an ignore replica failure, we need to make sure to bubble up the failure // so we will fail the shard if (!ignoreReplicaException(e)) { throw e; } } } else { throw new IllegalStateException("Unexpected index operation: " + item.request()); } } processAfter(request.refresh(), indexShard, location); }
@Override protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary( ClusterState clusterState, PrimaryOperationRequest shardRequest) { final BulkShardRequest request = shardRequest.request; final IndexService indexService = indicesService.indexServiceSafe(request.index()); final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id()); long[] preVersions = new long[request.items().length]; VersionType[] preVersionTypes = new VersionType[request.items().length]; Translog.Location location = null; for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) { BulkItemRequest item = request.items()[requestIndex]; if (item.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) item.request(); preVersions[requestIndex] = indexRequest.version(); preVersionTypes[requestIndex] = indexRequest.versionType(); try { WriteResult<IndexResponse> result = shardIndexOperation(request, indexRequest, clusterState, indexShard, true); location = locationToSync(location, result.location); // add the response IndexResponse indexResponse = result.response(); setResponse( item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse)); } catch (Throwable e) { // rethrow the failure if we are going to retry on primary and let parent failure to // handle it if (retryPrimaryException(e)) { // restore updated versions... for (int j = 0; j < requestIndex; j++) { applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); } throw (ElasticsearchException) e; } if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) { logger.trace( "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest); } else { logger.debug( "{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest); } // if its a conflict failure, and we already executed the request on a primary (and we // execute it // again, due to primary relocation and only processing up to N bulk items when the shard // gets closed) // then just use the response we got from the successful execution if (item.getPrimaryResponse() != null && isConflictException(e)) { setResponse(item, item.getPrimaryResponse()); } else { setResponse( item, new BulkItemResponse( item.id(), indexRequest.opType().lowercase(), new BulkItemResponse.Failure( request.index(), indexRequest.type(), indexRequest.id(), e))); } } } else if (item.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) item.request(); preVersions[requestIndex] = deleteRequest.version(); preVersionTypes[requestIndex] = deleteRequest.versionType(); try { // add the response final WriteResult<DeleteResponse> writeResult = shardDeleteOperation(request, deleteRequest, indexShard); DeleteResponse deleteResponse = writeResult.response(); location = locationToSync(location, writeResult.location); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse)); } catch (Throwable e) { // rethrow the failure if we are going to retry on primary and let parent failure to // handle it if (retryPrimaryException(e)) { // restore updated versions... for (int j = 0; j < requestIndex; j++) { applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); } throw (ElasticsearchException) e; } if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) { logger.trace( "{} failed to execute bulk item (delete) {}", e, shardRequest.shardId, deleteRequest); } else { logger.debug( "{} failed to execute bulk item (delete) {}", e, shardRequest.shardId, deleteRequest); } // if its a conflict failure, and we already executed the request on a primary (and we // execute it // again, due to primary relocation and only processing up to N bulk items when the shard // gets closed) // then just use the response we got from the successful execution if (item.getPrimaryResponse() != null && isConflictException(e)) { setResponse(item, item.getPrimaryResponse()); } else { setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_DELETE, new BulkItemResponse.Failure( request.index(), deleteRequest.type(), deleteRequest.id(), e))); } } } else if (item.request() instanceof UpdateRequest) { UpdateRequest updateRequest = (UpdateRequest) item.request(); preVersions[requestIndex] = updateRequest.version(); preVersionTypes[requestIndex] = updateRequest.versionType(); // We need to do the requested retries plus the initial attempt. We don't do < // 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE for (int updateAttemptsCount = 0; updateAttemptsCount <= updateRequest.retryOnConflict(); updateAttemptsCount++) { UpdateResult updateResult; try { updateResult = shardUpdateOperation(clusterState, request, updateRequest, indexShard); } catch (Throwable t) { updateResult = new UpdateResult(null, null, false, t, null); } if (updateResult.success()) { if (updateResult.writeResult != null) { location = locationToSync(location, updateResult.writeResult.location); } switch (updateResult.result.operation()) { case UPSERT: case INDEX: WriteResult<IndexResponse> result = updateResult.writeResult; IndexRequest indexRequest = updateResult.request(); BytesReference indexSourceAsBytes = indexRequest.source(); // add the response IndexResponse indexResponse = result.response(); UpdateResponse updateResponse = new UpdateResponse( indexResponse.getShardInfo(), indexResponse.getIndex(), indexResponse.getType(), indexResponse.getId(), indexResponse.getVersion(), indexResponse.isCreated()); if (updateRequest.fields() != null && updateRequest.fields().length > 0) { Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true); updateResponse.setGetResult( updateHelper.extractGetResult( updateRequest, shardRequest.request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); } item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), indexRequest); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); break; case DELETE: WriteResult<DeleteResponse> writeResult = updateResult.writeResult; DeleteResponse response = writeResult.response(); DeleteRequest deleteRequest = updateResult.request(); updateResponse = new UpdateResponse( response.getShardInfo(), response.getIndex(), response.getType(), response.getId(), response.getVersion(), false); updateResponse.setGetResult( updateHelper.extractGetResult( updateRequest, shardRequest.request.index(), response.getVersion(), updateResult.result.updatedSourceAsMap(), updateResult.result.updateSourceContentType(), null)); // Replace the update request to the translated delete request to execute on the // replica. item = request.items()[requestIndex] = new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); break; case NONE: setResponse( item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult)); item.setIgnoreOnReplica(); // no need to go to the replica break; } // NOTE: Breaking out of the retry_on_conflict loop! break; } else if (updateResult.failure()) { Throwable t = updateResult.error; if (updateResult.retry) { // updateAttemptCount is 0 based and marks current attempt, if it's equal to // retryOnConflict we are going out of the iteration if (updateAttemptsCount >= updateRequest.retryOnConflict()) { setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure( request.index(), updateRequest.type(), updateRequest.id(), t))); } } else { // rethrow the failure if we are going to retry on primary and let parent failure to // handle it if (retryPrimaryException(t)) { // restore updated versions... for (int j = 0; j < requestIndex; j++) { applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); } throw (ElasticsearchException) t; } // if its a conflict failure, and we already executed the request on a primary (and we // execute it // again, due to primary relocation and only processing up to N bulk items when the // shard gets closed) // then just use the response we got from the successful execution if (item.getPrimaryResponse() != null && isConflictException(t)) { setResponse(item, item.getPrimaryResponse()); } else if (updateResult.result == null) { setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure( shardRequest.request.index(), updateRequest.type(), updateRequest.id(), t))); } else { switch (updateResult.result.operation()) { case UPSERT: case INDEX: IndexRequest indexRequest = updateResult.request(); if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) { logger.trace( "{} failed to execute bulk item (index) {}", t, shardRequest.shardId, indexRequest); } else { logger.debug( "{} failed to execute bulk item (index) {}", t, shardRequest.shardId, indexRequest); } setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure( request.index(), indexRequest.type(), indexRequest.id(), t))); break; case DELETE: DeleteRequest deleteRequest = updateResult.request(); if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) { logger.trace( "{} failed to execute bulk item (delete) {}", t, shardRequest.shardId, deleteRequest); } else { logger.debug( "{} failed to execute bulk item (delete) {}", t, shardRequest.shardId, deleteRequest); } setResponse( item, new BulkItemResponse( item.id(), OP_TYPE_DELETE, new BulkItemResponse.Failure( request.index(), deleteRequest.type(), deleteRequest.id(), t))); break; } } // NOTE: Breaking out of the retry_on_conflict loop! break; } } } } else { throw new IllegalStateException("Unexpected index operation: " + item.request()); } assert item.getPrimaryResponse() != null; assert preVersionTypes[requestIndex] != null; } processAfter(request.refresh(), indexShard, location); BulkItemResponse[] responses = new BulkItemResponse[request.items().length]; BulkItemRequest[] items = request.items(); for (int i = 0; i < items.length; i++) { responses[i] = items[i].getPrimaryResponse(); } return new Tuple<>( new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request); }
private void executeBulk( final BulkRequest bulkRequest, final long startTime, final ActionListener<BulkResponse> listener, final AtomicArray<BulkItemResponse> responses) { final ClusterState clusterState = clusterService.state(); // TODO use timeout to wait here if its blocked... clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE); final ConcreteIndices concreteIndices = new ConcreteIndices(clusterState, indexNameExpressionResolver); MetaData metaData = clusterState.metaData(); for (int i = 0; i < bulkRequest.requests.size(); i++) { ActionRequest request = bulkRequest.requests.get(i); if (request instanceof DocumentRequest) { DocumentRequest req = (DocumentRequest) request; if (addFailureIfIndexIsUnavailable( req, bulkRequest, responses, i, concreteIndices, metaData)) { continue; } String concreteIndex = concreteIndices.resolveIfAbsent(req); if (request instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request; MappingMetaData mappingMd = null; if (metaData.hasIndex(concreteIndex)) { mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type()); } try { indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex); } catch (ElasticsearchParseException | RoutingMissingException e) { BulkItemResponse.Failure failure = new BulkItemResponse.Failure( concreteIndex, indexRequest.type(), indexRequest.id(), e); BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure); responses.set(i, bulkItemResponse); // make sure the request gets never processed again bulkRequest.requests.set(i, null); } } else { concreteIndices.resolveIfAbsent(req); req.routing( clusterState .metaData() .resolveIndexRouting(req.parent(), req.routing(), req.index())); } } } // first, go over all the requests and create a ShardId -> Operations mapping Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>(); for (int i = 0; i < bulkRequest.requests.size(); i++) { ActionRequest request = bulkRequest.requests.get(i); if (request instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request; String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index()); ShardId shardId = clusterService .operationRouting() .indexShards( clusterState, concreteIndex, indexRequest.type(), indexRequest.id(), indexRequest.routing()) .shardId(); List<BulkItemRequest> list = requestsByShard.get(shardId); if (list == null) { list = new ArrayList<>(); requestsByShard.put(shardId, list); } list.add(new BulkItemRequest(i, request)); } else if (request instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request; String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index()); MappingMetaData mappingMd = clusterState.metaData().index(concreteIndex).mappingOrDefault(deleteRequest.type()); if (mappingMd != null && mappingMd.routing().required() && deleteRequest.routing() == null) { // if routing is required, and no routing on the delete request, we need to broadcast // it.... GroupShardsIterator groupShards = clusterService.operationRouting().broadcastDeleteShards(clusterState, concreteIndex); for (ShardIterator shardIt : groupShards) { List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId()); if (list == null) { list = new ArrayList<>(); requestsByShard.put(shardIt.shardId(), list); } list.add(new BulkItemRequest(i, deleteRequest)); } } else { ShardId shardId = clusterService .operationRouting() .indexShards( clusterState, concreteIndex, deleteRequest.type(), deleteRequest.id(), deleteRequest.routing()) .shardId(); List<BulkItemRequest> list = requestsByShard.get(shardId); if (list == null) { list = new ArrayList<>(); requestsByShard.put(shardId, list); } list.add(new BulkItemRequest(i, request)); } } else if (request instanceof UpdateRequest) { UpdateRequest updateRequest = (UpdateRequest) request; String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index()); MappingMetaData mappingMd = clusterState.metaData().index(concreteIndex).mappingOrDefault(updateRequest.type()); if (mappingMd != null && mappingMd.routing().required() && updateRequest.routing() == null) { BulkItemResponse.Failure failure = new BulkItemResponse.Failure( updateRequest.index(), updateRequest.type(), updateRequest.id(), new IllegalArgumentException("routing is required for this item")); responses.set(i, new BulkItemResponse(i, updateRequest.type(), failure)); continue; } ShardId shardId = clusterService .operationRouting() .indexShards( clusterState, concreteIndex, updateRequest.type(), updateRequest.id(), updateRequest.routing()) .shardId(); List<BulkItemRequest> list = requestsByShard.get(shardId); if (list == null) { list = new ArrayList<>(); requestsByShard.put(shardId, list); } list.add(new BulkItemRequest(i, request)); } } if (requestsByShard.isEmpty()) { listener.onResponse( new BulkResponse( responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTime))); return; } final AtomicInteger counter = new AtomicInteger(requestsByShard.size()); for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) { final ShardId shardId = entry.getKey(); final List<BulkItemRequest> requests = entry.getValue(); BulkShardRequest bulkShardRequest = new BulkShardRequest( bulkRequest, shardId, bulkRequest.refresh(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel()); bulkShardRequest.timeout(bulkRequest.timeout()); shardBulkAction.execute( bulkShardRequest, new ActionListener<BulkShardResponse>() { @Override public void onResponse(BulkShardResponse bulkShardResponse) { for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) { // we may have no response if item failed if (bulkItemResponse.getResponse() != null) { bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo()); } responses.set(bulkItemResponse.getItemId(), bulkItemResponse); } if (counter.decrementAndGet() == 0) { finishHim(); } } @Override public void onFailure(Throwable e) { // create failures for all relevant requests for (BulkItemRequest request : requests) { if (request.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request.request(); responses.set( request.id(), new BulkItemResponse( request.id(), indexRequest.opType().toString().toLowerCase(Locale.ENGLISH), new BulkItemResponse.Failure( concreteIndices.getConcreteIndex(indexRequest.index()), indexRequest.type(), indexRequest.id(), e))); } else if (request.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request.request(); responses.set( request.id(), new BulkItemResponse( request.id(), "delete", new BulkItemResponse.Failure( concreteIndices.getConcreteIndex(deleteRequest.index()), deleteRequest.type(), deleteRequest.id(), e))); } else if (request.request() instanceof UpdateRequest) { UpdateRequest updateRequest = (UpdateRequest) request.request(); responses.set( request.id(), new BulkItemResponse( request.id(), "update", new BulkItemResponse.Failure( concreteIndices.getConcreteIndex(updateRequest.index()), updateRequest.type(), updateRequest.id(), e))); } } if (counter.decrementAndGet() == 0) { finishHim(); } } private void finishHim() { listener.onResponse( new BulkResponse( responses.toArray(new BulkItemResponse[responses.length()]), buildTookInMillis(startTime))); } }); } }
private void executeBulk( final BulkRequest bulkRequest, final long startTime, final ActionListener<BulkResponse> listener) { ClusterState clusterState = clusterService.state(); for (ActionRequest request : bulkRequest.requests) { if (request instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request; indexRequest.routing( clusterState .metaData() .resolveIndexRouting(indexRequest.routing(), indexRequest.index())); indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index())); if (allowIdGeneration) { if (indexRequest.id() == null) { indexRequest.id(UUID.randomBase64UUID()); // since we generate the id, change it to CREATE indexRequest.opType(IndexRequest.OpType.CREATE); } } } else if (request instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request; deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index())); } } final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()]; // first, go over all the requests and create a ShardId -> Operations mapping Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap(); for (int i = 0; i < bulkRequest.requests.size(); i++) { ActionRequest request = bulkRequest.requests.get(i); if (request instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request; // handle routing MappingMetaData mappingMd = clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type()); if (mappingMd != null) { try { indexRequest.processRouting(mappingMd); } catch (ElasticSearchException e) { responses[i] = new BulkItemResponse( i, indexRequest.opType().toString().toLowerCase(), new BulkItemResponse.Failure( indexRequest.index(), indexRequest.type(), indexRequest.id(), e.getDetailedMessage())); continue; } } ShardId shardId = clusterService .operationRouting() .indexShards( clusterState, indexRequest.index(), indexRequest.type(), indexRequest.id(), indexRequest.routing()) .shardId(); List<BulkItemRequest> list = requestsByShard.get(shardId); if (list == null) { list = Lists.newArrayList(); requestsByShard.put(shardId, list); } list.add(new BulkItemRequest(i, request)); } else if (request instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request; MappingMetaData mappingMd = clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type()); if (mappingMd != null && mappingMd.routing().required() && deleteRequest.routing() == null) { // if routing is required, and no routing on the delete request, we need to broadcast // it.... GroupShardsIterator groupShards = clusterService .operationRouting() .broadcastDeleteShards(clusterState, deleteRequest.index()); for (ShardIterator shardIt : groupShards) { List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId()); if (list == null) { list = Lists.newArrayList(); requestsByShard.put(shardIt.shardId(), list); } list.add(new BulkItemRequest(i, request)); } } else { ShardId shardId = clusterService .operationRouting() .deleteShards( clusterState, deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), deleteRequest.routing()) .shardId(); List<BulkItemRequest> list = requestsByShard.get(shardId); if (list == null) { list = Lists.newArrayList(); requestsByShard.put(shardId, list); } list.add(new BulkItemRequest(i, request)); } } } if (requestsByShard.isEmpty()) { listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime)); return; } final AtomicInteger counter = new AtomicInteger(requestsByShard.size()); for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) { final ShardId shardId = entry.getKey(); final List<BulkItemRequest> requests = entry.getValue(); BulkShardRequest bulkShardRequest = new BulkShardRequest( shardId.index().name(), shardId.id(), bulkRequest.refresh(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.replicationType(bulkRequest.replicationType()); bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel()); shardBulkAction.execute( bulkShardRequest, new ActionListener<BulkShardResponse>() { @Override public void onResponse(BulkShardResponse bulkShardResponse) { synchronized (responses) { for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) { responses[bulkItemResponse.itemId()] = bulkItemResponse; } } if (counter.decrementAndGet() == 0) { finishHim(); } } @Override public void onFailure(Throwable e) { // create failures for all relevant requests String message = ExceptionsHelper.detailedMessage(e); synchronized (responses) { for (BulkItemRequest request : requests) { if (request.request() instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) request.request(); responses[request.id()] = new BulkItemResponse( request.id(), indexRequest.opType().toString().toLowerCase(), new BulkItemResponse.Failure( indexRequest.index(), indexRequest.type(), indexRequest.id(), message)); } else if (request.request() instanceof DeleteRequest) { DeleteRequest deleteRequest = (DeleteRequest) request.request(); responses[request.id()] = new BulkItemResponse( request.id(), "delete", new BulkItemResponse.Failure( deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), message)); } } } if (counter.decrementAndGet() == 0) { finishHim(); } } private void finishHim() { listener.onResponse( new BulkResponse(responses, System.currentTimeMillis() - startTime)); } }); } }