private WriteResult<DeleteResponse> shardDeleteOperation(
BulkShardRequest request, DeleteRequest deleteRequest, IndexShard indexShard) {
Engine.Delete delete =
indexShard.prepareDelete(
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.version(),
deleteRequest.versionType(),
Engine.Operation.Origin.PRIMARY);
indexShard.delete(delete);
// update the request with the version so it will go to the replicas
deleteRequest.versionType(delete.versionType().versionTypeForReplicationAndRecovery());
deleteRequest.version(delete.version());
assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version());
DeleteResponse deleteResponse =
new DeleteResponse(
request.index(),
deleteRequest.type(),
deleteRequest.id(),
delete.version(),
delete.found());
return new WriteResult(deleteResponse, delete.getTranslogLocation());
}
public DeleteRequest(DeleteRequest request) {
super(request);
this.type = request.type();
this.id = request.id();
this.routing = request.routing();
this.refresh = request.refresh();
this.version = request.version();
this.versionType = request.versionType();
}
@Override
protected void shardOperationOnReplica(ReplicaOperationRequest shardRequest) {
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.request.index())
.shardSafe(shardRequest.shardId);
final IngestShardRequest request = shardRequest.request;
int size = request.items().size();
for (int i = 0; i < size; i++) {
IngestItemRequest item = request.items().get(i);
if (item == null) {
continue;
}
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
try {
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
.type(indexRequest.type())
.id(indexRequest.id())
.routing(indexRequest.routing())
.parent(indexRequest.parent())
.timestamp(indexRequest.timestamp())
.ttl(indexRequest.ttl());
if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard
.prepareIndex(sourceToParse)
.version(indexRequest.version())
.origin(Engine.Operation.Origin.REPLICA);
indexShard.index(index);
} else {
Engine.Create create =
indexShard
.prepareCreate(sourceToParse)
.version(indexRequest.version())
.origin(Engine.Operation.Origin.REPLICA);
indexShard.create(create);
}
} catch (Exception e) {
// ignore, we are on backup
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
try {
Engine.Delete delete =
indexShard
.prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
.origin(Engine.Operation.Origin.REPLICA);
indexShard.delete(delete);
} catch (Exception e) {
// ignore, we are on backup
}
}
}
}
private boolean setResponseFailureIfIndexMatches(
AtomicArray<BulkItemResponse> responses,
int idx,
ActionRequest request,
String index,
Throwable e) {
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
if (index.equals(indexRequest.index())) {
responses.set(
idx,
new BulkItemResponse(
idx,
"index",
new BulkItemResponse.Failure(
indexRequest.index(), indexRequest.type(), indexRequest.id(), e)));
return true;
}
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
if (index.equals(deleteRequest.index())) {
responses.set(
idx,
new BulkItemResponse(
idx,
"delete",
new BulkItemResponse.Failure(
deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), e)));
return true;
}
} else if (request instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) request;
if (index.equals(updateRequest.index())) {
responses.set(
idx,
new BulkItemResponse(
idx,
"update",
new BulkItemResponse.Failure(
updateRequest.index(), updateRequest.type(), updateRequest.id(), e)));
return true;
}
} else {
throw new ElasticsearchException(
"Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
}
return false;
}
@Override
protected PrimaryResponse<IngestShardResponse, IngestShardRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
final IngestShardRequest request = shardRequest.request;
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.request.index())
.shardSafe(shardRequest.shardId);
int successSize = 0;
List<IngestItemFailure> failure = newLinkedList();
int size = request.items().size();
long[] versions = new long[size];
Set<Tuple<String, String>> mappingsToUpdate = newHashSet();
for (int i = 0; i < size; i++) {
IngestItemRequest item = request.items().get(i);
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
Engine.IndexingOperation op = null;
try {
// validate, if routing is required, that we got routing
MappingMetaData mappingMd =
clusterState.metaData().index(request.index()).mappingOrDefault(indexRequest.type());
if (mappingMd != null && mappingMd.routing().required()) {
if (indexRequest.routing() == null) {
throw new RoutingMissingException(
indexRequest.index(), indexRequest.type(), indexRequest.id());
}
}
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.PRIMARY, indexRequest.source())
.type(indexRequest.type())
.id(indexRequest.id())
.routing(indexRequest.routing())
.parent(indexRequest.parent())
.timestamp(indexRequest.timestamp())
.ttl(indexRequest.ttl());
long version;
if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard
.prepareIndex(sourceToParse)
.version(indexRequest.version())
.versionType(indexRequest.versionType())
.origin(Engine.Operation.Origin.PRIMARY);
op = index;
indexShard.index(index);
version = index.version();
} else {
Engine.Create create =
indexShard
.prepareCreate(sourceToParse)
.version(indexRequest.version())
.versionType(indexRequest.versionType())
.origin(Engine.Operation.Origin.PRIMARY);
op = create;
indexShard.create(create);
version = create.version();
}
versions[i] = indexRequest.version();
// update the version on request so it will happen on the replicas
indexRequest.version(version);
successSize++;
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < i; j++) {
applyVersion(request.items().get(j), versions[j]);
}
logger.error(e.getMessage(), e);
throw new ElasticsearchException(e.getMessage());
}
if (e instanceof ElasticsearchException
&& ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
logger.error(
"[{}][{}] failed to execute bulk item (index) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
indexRequest);
} else {
logger.error(
"[{}][{}] failed to execute bulk item (index) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
indexRequest);
}
failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
// nullify the request so it won't execute on the replicas
request.items().set(i, null);
} finally {
// update mapping on master if needed, we won't update changes to the same type, since
// once its changed, it won't have mappers added
if (op != null && op.parsedDoc().mappingsModified()) {
mappingsToUpdate.add(Tuple.tuple(indexRequest.index(), indexRequest.type()));
}
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
try {
Engine.Delete delete =
indexShard
.prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
.versionType(deleteRequest.versionType())
.origin(Engine.Operation.Origin.PRIMARY);
indexShard.delete(delete);
// update the request with teh version so it will go to the replicas
deleteRequest.version(delete.version());
successSize++;
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < i; j++) {
applyVersion(request.items().get(j), versions[j]);
}
logger.error(e.getMessage(), e);
throw new ElasticsearchException(e.getMessage());
}
if (e instanceof ElasticsearchException
&& ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
logger.trace(
"[{}][{}] failed to execute bulk item (delete) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
deleteRequest);
} else {
logger.debug(
"[{}][{}] failed to execute bulk item (delete) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
deleteRequest);
}
failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
// nullify the request so it won't execute on the replicas
request.items().set(i, null);
}
}
}
if (!mappingsToUpdate.isEmpty()) {
for (Tuple<String, String> mappingToUpdate : mappingsToUpdate) {
logger.info("mapping update {} {}", mappingToUpdate.v1(), mappingToUpdate.v2());
updateMappingOnMaster(mappingToUpdate.v1(), mappingToUpdate.v2());
}
}
IngestShardResponse response =
new IngestShardResponse(
new ShardId(request.index(), request.shardId()), successSize, failure);
return new PrimaryResponse<IngestShardResponse, IngestShardRequest>(
shardRequest.request, response, null);
}
@Override
protected void shardOperationOnReplica(ShardId shardId, BulkShardRequest request) {
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardId.id());
Translog.Location location = null;
for (int i = 0; i < request.items().length; i++) {
BulkItemRequest item = request.items()[i];
if (item == null || item.isIgnoreOnReplica()) {
continue;
}
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
try {
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.REPLICA, indexRequest.source())
.index(shardId.getIndex())
.type(indexRequest.type())
.id(indexRequest.id())
.routing(indexRequest.routing())
.parent(indexRequest.parent())
.timestamp(indexRequest.timestamp())
.ttl(indexRequest.ttl());
final Engine.IndexingOperation operation;
if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
operation =
indexShard.prepareIndex(
sourceToParse,
indexRequest.version(),
indexRequest.versionType(),
Engine.Operation.Origin.REPLICA,
request.canHaveDuplicates() || indexRequest.canHaveDuplicates());
} else {
assert indexRequest.opType() == IndexRequest.OpType.CREATE : indexRequest.opType();
operation =
indexShard.prepareCreate(
sourceToParse,
indexRequest.version(),
indexRequest.versionType(),
Engine.Operation.Origin.REPLICA,
request.canHaveDuplicates() || indexRequest.canHaveDuplicates(),
indexRequest.autoGeneratedId());
}
Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
if (update != null) {
throw new RetryOnReplicaException(
shardId,
"Mappings are not available on the replica yet, triggered update: " + update);
}
operation.execute(indexShard);
location = locationToSync(location, operation.getTranslogLocation());
} catch (Throwable e) {
// if its not an ignore replica failure, we need to make sure to bubble up the failure
// so we will fail the shard
if (!ignoreReplicaException(e)) {
throw e;
}
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
try {
Engine.Delete delete =
indexShard.prepareDelete(
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.version(),
deleteRequest.versionType(),
Engine.Operation.Origin.REPLICA);
indexShard.delete(delete);
location = locationToSync(location, delete.getTranslogLocation());
} catch (Throwable e) {
// if its not an ignore replica failure, we need to make sure to bubble up the failure
// so we will fail the shard
if (!ignoreReplicaException(e)) {
throw e;
}
}
} else {
throw new IllegalStateException("Unexpected index operation: " + item.request());
}
}
processAfter(request.refresh(), indexShard, location);
}
@Override
protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
final BulkShardRequest request = shardRequest.request;
final IndexService indexService = indicesService.indexServiceSafe(request.index());
final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
long[] preVersions = new long[request.items().length];
VersionType[] preVersionTypes = new VersionType[request.items().length];
Translog.Location location = null;
for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
BulkItemRequest item = request.items()[requestIndex];
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
preVersions[requestIndex] = indexRequest.version();
preVersionTypes[requestIndex] = indexRequest.versionType();
try {
WriteResult<IndexResponse> result =
shardIndexOperation(request, indexRequest, clusterState, indexShard, true);
location = locationToSync(location, result.location);
// add the response
IndexResponse indexResponse = result.response();
setResponse(
item,
new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < requestIndex; j++) {
applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
}
throw (ElasticsearchException) e;
}
if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
} else {
logger.debug(
"{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
}
// if its a conflict failure, and we already executed the request on a primary (and we
// execute it
// again, due to primary relocation and only processing up to N bulk items when the shard
// gets closed)
// then just use the response we got from the successful execution
if (item.getPrimaryResponse() != null && isConflictException(e)) {
setResponse(item, item.getPrimaryResponse());
} else {
setResponse(
item,
new BulkItemResponse(
item.id(),
indexRequest.opType().lowercase(),
new BulkItemResponse.Failure(
request.index(), indexRequest.type(), indexRequest.id(), e)));
}
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
preVersions[requestIndex] = deleteRequest.version();
preVersionTypes[requestIndex] = deleteRequest.versionType();
try {
// add the response
final WriteResult<DeleteResponse> writeResult =
shardDeleteOperation(request, deleteRequest, indexShard);
DeleteResponse deleteResponse = writeResult.response();
location = locationToSync(location, writeResult.location);
setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < requestIndex; j++) {
applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
}
throw (ElasticsearchException) e;
}
if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (delete) {}",
e,
shardRequest.shardId,
deleteRequest);
} else {
logger.debug(
"{} failed to execute bulk item (delete) {}",
e,
shardRequest.shardId,
deleteRequest);
}
// if its a conflict failure, and we already executed the request on a primary (and we
// execute it
// again, due to primary relocation and only processing up to N bulk items when the shard
// gets closed)
// then just use the response we got from the successful execution
if (item.getPrimaryResponse() != null && isConflictException(e)) {
setResponse(item, item.getPrimaryResponse());
} else {
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_DELETE,
new BulkItemResponse.Failure(
request.index(), deleteRequest.type(), deleteRequest.id(), e)));
}
}
} else if (item.request() instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) item.request();
preVersions[requestIndex] = updateRequest.version();
preVersionTypes[requestIndex] = updateRequest.versionType();
// We need to do the requested retries plus the initial attempt. We don't do <
// 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE
for (int updateAttemptsCount = 0;
updateAttemptsCount <= updateRequest.retryOnConflict();
updateAttemptsCount++) {
UpdateResult updateResult;
try {
updateResult = shardUpdateOperation(clusterState, request, updateRequest, indexShard);
} catch (Throwable t) {
updateResult = new UpdateResult(null, null, false, t, null);
}
if (updateResult.success()) {
if (updateResult.writeResult != null) {
location = locationToSync(location, updateResult.writeResult.location);
}
switch (updateResult.result.operation()) {
case UPSERT:
case INDEX:
WriteResult<IndexResponse> result = updateResult.writeResult;
IndexRequest indexRequest = updateResult.request();
BytesReference indexSourceAsBytes = indexRequest.source();
// add the response
IndexResponse indexResponse = result.response();
UpdateResponse updateResponse =
new UpdateResponse(
indexResponse.getShardInfo(),
indexResponse.getIndex(),
indexResponse.getType(),
indexResponse.getId(),
indexResponse.getVersion(),
indexResponse.isCreated());
if (updateRequest.fields() != null && updateRequest.fields().length > 0) {
Tuple<XContentType, Map<String, Object>> sourceAndContent =
XContentHelper.convertToMap(indexSourceAsBytes, true);
updateResponse.setGetResult(
updateHelper.extractGetResult(
updateRequest,
shardRequest.request.index(),
indexResponse.getVersion(),
sourceAndContent.v2(),
sourceAndContent.v1(),
indexSourceAsBytes));
}
item =
request.items()[requestIndex] =
new BulkItemRequest(request.items()[requestIndex].id(), indexRequest);
setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
break;
case DELETE:
WriteResult<DeleteResponse> writeResult = updateResult.writeResult;
DeleteResponse response = writeResult.response();
DeleteRequest deleteRequest = updateResult.request();
updateResponse =
new UpdateResponse(
response.getShardInfo(),
response.getIndex(),
response.getType(),
response.getId(),
response.getVersion(),
false);
updateResponse.setGetResult(
updateHelper.extractGetResult(
updateRequest,
shardRequest.request.index(),
response.getVersion(),
updateResult.result.updatedSourceAsMap(),
updateResult.result.updateSourceContentType(),
null));
// Replace the update request to the translated delete request to execute on the
// replica.
item =
request.items()[requestIndex] =
new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest);
setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
break;
case NONE:
setResponse(
item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult));
item.setIgnoreOnReplica(); // no need to go to the replica
break;
}
// NOTE: Breaking out of the retry_on_conflict loop!
break;
} else if (updateResult.failure()) {
Throwable t = updateResult.error;
if (updateResult.retry) {
// updateAttemptCount is 0 based and marks current attempt, if it's equal to
// retryOnConflict we are going out of the iteration
if (updateAttemptsCount >= updateRequest.retryOnConflict()) {
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_UPDATE,
new BulkItemResponse.Failure(
request.index(), updateRequest.type(), updateRequest.id(), t)));
}
} else {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(t)) {
// restore updated versions...
for (int j = 0; j < requestIndex; j++) {
applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
}
throw (ElasticsearchException) t;
}
// if its a conflict failure, and we already executed the request on a primary (and we
// execute it
// again, due to primary relocation and only processing up to N bulk items when the
// shard gets closed)
// then just use the response we got from the successful execution
if (item.getPrimaryResponse() != null && isConflictException(t)) {
setResponse(item, item.getPrimaryResponse());
} else if (updateResult.result == null) {
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_UPDATE,
new BulkItemResponse.Failure(
shardRequest.request.index(),
updateRequest.type(),
updateRequest.id(),
t)));
} else {
switch (updateResult.result.operation()) {
case UPSERT:
case INDEX:
IndexRequest indexRequest = updateResult.request();
if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (index) {}",
t,
shardRequest.shardId,
indexRequest);
} else {
logger.debug(
"{} failed to execute bulk item (index) {}",
t,
shardRequest.shardId,
indexRequest);
}
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_UPDATE,
new BulkItemResponse.Failure(
request.index(), indexRequest.type(), indexRequest.id(), t)));
break;
case DELETE:
DeleteRequest deleteRequest = updateResult.request();
if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (delete) {}",
t,
shardRequest.shardId,
deleteRequest);
} else {
logger.debug(
"{} failed to execute bulk item (delete) {}",
t,
shardRequest.shardId,
deleteRequest);
}
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_DELETE,
new BulkItemResponse.Failure(
request.index(), deleteRequest.type(), deleteRequest.id(), t)));
break;
}
}
// NOTE: Breaking out of the retry_on_conflict loop!
break;
}
}
}
} else {
throw new IllegalStateException("Unexpected index operation: " + item.request());
}
assert item.getPrimaryResponse() != null;
assert preVersionTypes[requestIndex] != null;
}
processAfter(request.refresh(), indexShard, location);
BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
BulkItemRequest[] items = request.items();
for (int i = 0; i < items.length; i++) {
responses[i] = items[i].getPrimaryResponse();
}
return new Tuple<>(
new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request);
}
private void executeBulk(
final BulkRequest bulkRequest,
final long startTime,
final ActionListener<BulkResponse> listener,
final AtomicArray<BulkItemResponse> responses) {
final ClusterState clusterState = clusterService.state();
// TODO use timeout to wait here if its blocked...
clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);
final ConcreteIndices concreteIndices =
new ConcreteIndices(clusterState, indexNameExpressionResolver);
MetaData metaData = clusterState.metaData();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof DocumentRequest) {
DocumentRequest req = (DocumentRequest) request;
if (addFailureIfIndexIsUnavailable(
req, bulkRequest, responses, i, concreteIndices, metaData)) {
continue;
}
String concreteIndex = concreteIndices.resolveIfAbsent(req);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
MappingMetaData mappingMd = null;
if (metaData.hasIndex(concreteIndex)) {
mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type());
}
try {
indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
} catch (ElasticsearchParseException | RoutingMissingException e) {
BulkItemResponse.Failure failure =
new BulkItemResponse.Failure(
concreteIndex, indexRequest.type(), indexRequest.id(), e);
BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
responses.set(i, bulkItemResponse);
// make sure the request gets never processed again
bulkRequest.requests.set(i, null);
}
} else {
concreteIndices.resolveIfAbsent(req);
req.routing(
clusterState
.metaData()
.resolveIndexRouting(req.parent(), req.routing(), req.index()));
}
}
}
// first, go over all the requests and create a ShardId -> Operations mapping
Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index());
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
concreteIndex,
indexRequest.type(),
indexRequest.id(),
indexRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index());
MappingMetaData mappingMd =
clusterState.metaData().index(concreteIndex).mappingOrDefault(deleteRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& deleteRequest.routing() == null) {
// if routing is required, and no routing on the delete request, we need to broadcast
// it....
GroupShardsIterator groupShards =
clusterService.operationRouting().broadcastDeleteShards(clusterState, concreteIndex);
for (ShardIterator shardIt : groupShards) {
List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardIt.shardId(), list);
}
list.add(new BulkItemRequest(i, deleteRequest));
}
} else {
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
concreteIndex,
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
} else if (request instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) request;
String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index());
MappingMetaData mappingMd =
clusterState.metaData().index(concreteIndex).mappingOrDefault(updateRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& updateRequest.routing() == null) {
BulkItemResponse.Failure failure =
new BulkItemResponse.Failure(
updateRequest.index(),
updateRequest.type(),
updateRequest.id(),
new IllegalArgumentException("routing is required for this item"));
responses.set(i, new BulkItemResponse(i, updateRequest.type(), failure));
continue;
}
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
concreteIndex,
updateRequest.type(),
updateRequest.id(),
updateRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
}
if (requestsByShard.isEmpty()) {
listener.onResponse(
new BulkResponse(
responses.toArray(new BulkItemResponse[responses.length()]),
buildTookInMillis(startTime)));
return;
}
final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
final ShardId shardId = entry.getKey();
final List<BulkItemRequest> requests = entry.getValue();
BulkShardRequest bulkShardRequest =
new BulkShardRequest(
bulkRequest,
shardId,
bulkRequest.refresh(),
requests.toArray(new BulkItemRequest[requests.size()]));
bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
bulkShardRequest.timeout(bulkRequest.timeout());
shardBulkAction.execute(
bulkShardRequest,
new ActionListener<BulkShardResponse>() {
@Override
public void onResponse(BulkShardResponse bulkShardResponse) {
for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
// we may have no response if item failed
if (bulkItemResponse.getResponse() != null) {
bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
}
responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
@Override
public void onFailure(Throwable e) {
// create failures for all relevant requests
for (BulkItemRequest request : requests) {
if (request.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request.request();
responses.set(
request.id(),
new BulkItemResponse(
request.id(),
indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
new BulkItemResponse.Failure(
concreteIndices.getConcreteIndex(indexRequest.index()),
indexRequest.type(),
indexRequest.id(),
e)));
} else if (request.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request.request();
responses.set(
request.id(),
new BulkItemResponse(
request.id(),
"delete",
new BulkItemResponse.Failure(
concreteIndices.getConcreteIndex(deleteRequest.index()),
deleteRequest.type(),
deleteRequest.id(),
e)));
} else if (request.request() instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) request.request();
responses.set(
request.id(),
new BulkItemResponse(
request.id(),
"update",
new BulkItemResponse.Failure(
concreteIndices.getConcreteIndex(updateRequest.index()),
updateRequest.type(),
updateRequest.id(),
e)));
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
private void finishHim() {
listener.onResponse(
new BulkResponse(
responses.toArray(new BulkItemResponse[responses.length()]),
buildTookInMillis(startTime)));
}
});
}
}
private void executeBulk(
final BulkRequest bulkRequest,
final long startTime,
final ActionListener<BulkResponse> listener) {
ClusterState clusterState = clusterService.state();
for (ActionRequest request : bulkRequest.requests) {
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
indexRequest.routing(
clusterState
.metaData()
.resolveIndexRouting(indexRequest.routing(), indexRequest.index()));
indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index()));
if (allowIdGeneration) {
if (indexRequest.id() == null) {
indexRequest.id(UUID.randomBase64UUID());
// since we generate the id, change it to CREATE
indexRequest.opType(IndexRequest.OpType.CREATE);
}
}
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index()));
}
}
final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()];
// first, go over all the requests and create a ShardId -> Operations mapping
Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
// handle routing
MappingMetaData mappingMd =
clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type());
if (mappingMd != null) {
try {
indexRequest.processRouting(mappingMd);
} catch (ElasticSearchException e) {
responses[i] =
new BulkItemResponse(
i,
indexRequest.opType().toString().toLowerCase(),
new BulkItemResponse.Failure(
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
e.getDetailedMessage()));
continue;
}
}
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
indexRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
MappingMetaData mappingMd =
clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& deleteRequest.routing() == null) {
// if routing is required, and no routing on the delete request, we need to broadcast
// it....
GroupShardsIterator groupShards =
clusterService
.operationRouting()
.broadcastDeleteShards(clusterState, deleteRequest.index());
for (ShardIterator shardIt : groupShards) {
List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardIt.shardId(), list);
}
list.add(new BulkItemRequest(i, request));
}
} else {
ShardId shardId =
clusterService
.operationRouting()
.deleteShards(
clusterState,
deleteRequest.index(),
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
}
}
if (requestsByShard.isEmpty()) {
listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime));
return;
}
final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
final ShardId shardId = entry.getKey();
final List<BulkItemRequest> requests = entry.getValue();
BulkShardRequest bulkShardRequest =
new BulkShardRequest(
shardId.index().name(),
shardId.id(),
bulkRequest.refresh(),
requests.toArray(new BulkItemRequest[requests.size()]));
bulkShardRequest.replicationType(bulkRequest.replicationType());
bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
shardBulkAction.execute(
bulkShardRequest,
new ActionListener<BulkShardResponse>() {
@Override
public void onResponse(BulkShardResponse bulkShardResponse) {
synchronized (responses) {
for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) {
responses[bulkItemResponse.itemId()] = bulkItemResponse;
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
@Override
public void onFailure(Throwable e) {
// create failures for all relevant requests
String message = ExceptionsHelper.detailedMessage(e);
synchronized (responses) {
for (BulkItemRequest request : requests) {
if (request.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request.request();
responses[request.id()] =
new BulkItemResponse(
request.id(),
indexRequest.opType().toString().toLowerCase(),
new BulkItemResponse.Failure(
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
message));
} else if (request.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request.request();
responses[request.id()] =
new BulkItemResponse(
request.id(),
"delete",
new BulkItemResponse.Failure(
deleteRequest.index(),
deleteRequest.type(),
deleteRequest.id(),
message));
}
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
private void finishHim() {
listener.onResponse(
new BulkResponse(responses, System.currentTimeMillis() - startTime));
}
});
}
}