@Override
protected Tuple<IndexResponse, IndexRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) throws Throwable {
final IndexRequest request = shardRequest.request;
// validate, if routing is required, that we got routing
IndexMetaData indexMetaData = clusterState.metaData().index(shardRequest.shardId.getIndex());
MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type());
if (mappingMd != null && mappingMd.routing().required()) {
if (request.routing() == null) {
throw new RoutingMissingException(
shardRequest.shardId.getIndex(), request.type(), request.id());
}
}
IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
final WriteResult<IndexResponse> result =
executeIndexRequestOnPrimary(null, request, indexShard);
final IndexResponse response = result.response;
final Translog.Location location = result.location;
processAfter(request.refresh(), indexShard, location);
return new Tuple<>(response, shardRequest.request);
}
public void process(
@Nullable MappingMetaData mappingMd, boolean allowIdGeneration, String concreteIndex) {
if (mappingMd != null) {
// might as well check for routing here
if (mappingMd.routing().required() && routing == null) {
throw new RoutingMissingException(concreteIndex, type, id);
}
if (parent != null && !mappingMd.hasParentField()) {
throw new IllegalArgumentException(
"Can't specify parent if no parent field has been configured");
}
} else {
if (parent != null) {
throw new IllegalArgumentException(
"Can't specify parent if no parent field has been configured");
}
}
// generate id if not already provided and id generation is allowed
if (allowIdGeneration && id == null) {
assert autoGeneratedTimestamp == -1;
autoGeneratedTimestamp = Math.max(0, System.currentTimeMillis()); // extra paranoia
id(UUIDs.base64UUID());
}
}
private WriteResult shardIndexOperation(
BulkShardRequest request,
IndexRequest indexRequest,
ClusterState clusterState,
IndexShard indexShard,
boolean processed)
throws Throwable {
// validate, if routing is required, that we got routing
MappingMetaData mappingMd =
clusterState.metaData().index(request.index()).mappingOrDefault(indexRequest.type());
if (mappingMd != null && mappingMd.routing().required()) {
if (indexRequest.routing() == null) {
throw new RoutingMissingException(request.index(), indexRequest.type(), indexRequest.id());
}
}
if (!processed) {
indexRequest.process(clusterState.metaData(), mappingMd, allowIdGeneration, request.index());
}
return executeIndexRequestOnPrimary(request, indexRequest, indexShard);
}
@Override
protected PrimaryResponse<IndexResponse, IndexRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
final IndexRequest request = shardRequest.request;
// validate, if routing is required, that we got routing
IndexMetaData indexMetaData = clusterState.metaData().index(shardRequest.shardId.getIndex());
MappingMetaData mappingMd = indexMetaData.mappingOrDefault(request.type());
if (mappingMd != null && mappingMd.routing().required()) {
if (request.routing() == null) {
throw new RoutingMissingException(
shardRequest.shardId.getIndex(), request.type(), request.id());
}
}
IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId.getIndex());
IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source())
.type(request.type())
.id(request.id())
.routing(request.routing())
.parent(request.parent())
.timestamp(request.timestamp())
.ttl(request.ttl());
long version;
boolean created;
Engine.IndexingOperation op;
if (request.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard.prepareIndex(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.PRIMARY,
request.canHaveDuplicates());
if (index.parsedDoc().mappingsModified()) {
mappingUpdatedAction.updateMappingOnMaster(
shardRequest.shardId.getIndex(), index.docMapper(), indexService.indexUUID());
}
indexShard.index(index);
version = index.version();
op = index;
created = index.created();
} else {
Engine.Create create =
indexShard.prepareCreate(
sourceToParse,
request.version(),
request.versionType(),
Engine.Operation.Origin.PRIMARY,
request.canHaveDuplicates(),
request.autoGeneratedId());
if (create.parsedDoc().mappingsModified()) {
mappingUpdatedAction.updateMappingOnMaster(
shardRequest.shardId.getIndex(), create.docMapper(), indexService.indexUUID());
}
indexShard.create(create);
version = create.version();
op = create;
created = true;
}
if (request.refresh()) {
try {
indexShard.refresh(new Engine.Refresh("refresh_flag_index").force(false));
} catch (Throwable e) {
// ignore
}
}
// update the version on the request, so it will be used for the replicas
request.version(version);
request.versionType(request.versionType().versionTypeForReplicationAndRecovery());
assert request.versionType().validateVersionForWrites(request.version());
IndexResponse response =
new IndexResponse(
shardRequest.shardId.getIndex(), request.type(), request.id(), version, created);
return new PrimaryResponse<>(shardRequest.request, response, op);
}
public void process(
MetaData metaData,
String aliasOrIndex,
@Nullable MappingMetaData mappingMd,
boolean allowIdGeneration)
throws ElasticsearchException {
// resolve the routing if needed
routing(metaData.resolveIndexRouting(routing, aliasOrIndex));
// resolve timestamp if provided externally
if (timestamp != null) {
timestamp =
MappingMetaData.Timestamp.parseStringTimestamp(
timestamp,
mappingMd != null
? mappingMd.timestamp().dateTimeFormatter()
: TimestampFieldMapper.Defaults.DATE_TIME_FORMATTER);
}
// extract values if needed
if (mappingMd != null) {
MappingMetaData.ParseContext parseContext =
mappingMd.createParseContext(id, routing, timestamp);
if (parseContext.shouldParse()) {
XContentParser parser = null;
try {
parser = XContentHelper.createParser(source);
mappingMd.parse(parser, parseContext);
if (parseContext.shouldParseId()) {
id = parseContext.id();
}
if (parseContext.shouldParseRouting()) {
routing = parseContext.routing();
}
if (parseContext.shouldParseTimestamp()) {
timestamp = parseContext.timestamp();
timestamp =
MappingMetaData.Timestamp.parseStringTimestamp(
timestamp, mappingMd.timestamp().dateTimeFormatter());
}
} catch (Exception e) {
throw new ElasticsearchParseException(
"failed to parse doc to extract routing/timestamp", e);
} finally {
if (parser != null) {
parser.close();
}
}
}
// might as well check for routing here
if (mappingMd.routing().required() && routing == null) {
throw new RoutingMissingException(index, type, id);
}
if (parent != null && !mappingMd.hasParentField()) {
throw new ElasticsearchIllegalArgumentException(
"Can't specify parent if no parent field has been configured");
}
} else {
if (parent != null) {
throw new ElasticsearchIllegalArgumentException(
"Can't specify parent if no parent field has been configured");
}
}
// generate id if not already provided and id generation is allowed
if (allowIdGeneration) {
if (id == null) {
id(Strings.randomBase64UUID());
// since we generate the id, change it to CREATE
opType(IndexRequest.OpType.CREATE);
}
}
// generate timestamp if not provided, we always have one post this stage...
if (timestamp == null) {
timestamp = Long.toString(System.currentTimeMillis());
}
}
@Override
protected PrimaryResponse<IngestShardResponse, IngestShardRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
final IngestShardRequest request = shardRequest.request;
IndexShard indexShard =
indicesService
.indexServiceSafe(shardRequest.request.index())
.shardSafe(shardRequest.shardId);
int successSize = 0;
List<IngestItemFailure> failure = newLinkedList();
int size = request.items().size();
long[] versions = new long[size];
Set<Tuple<String, String>> mappingsToUpdate = newHashSet();
for (int i = 0; i < size; i++) {
IngestItemRequest item = request.items().get(i);
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
Engine.IndexingOperation op = null;
try {
// validate, if routing is required, that we got routing
MappingMetaData mappingMd =
clusterState.metaData().index(request.index()).mappingOrDefault(indexRequest.type());
if (mappingMd != null && mappingMd.routing().required()) {
if (indexRequest.routing() == null) {
throw new RoutingMissingException(
indexRequest.index(), indexRequest.type(), indexRequest.id());
}
}
SourceToParse sourceToParse =
SourceToParse.source(SourceToParse.Origin.PRIMARY, indexRequest.source())
.type(indexRequest.type())
.id(indexRequest.id())
.routing(indexRequest.routing())
.parent(indexRequest.parent())
.timestamp(indexRequest.timestamp())
.ttl(indexRequest.ttl());
long version;
if (indexRequest.opType() == IndexRequest.OpType.INDEX) {
Engine.Index index =
indexShard
.prepareIndex(sourceToParse)
.version(indexRequest.version())
.versionType(indexRequest.versionType())
.origin(Engine.Operation.Origin.PRIMARY);
op = index;
indexShard.index(index);
version = index.version();
} else {
Engine.Create create =
indexShard
.prepareCreate(sourceToParse)
.version(indexRequest.version())
.versionType(indexRequest.versionType())
.origin(Engine.Operation.Origin.PRIMARY);
op = create;
indexShard.create(create);
version = create.version();
}
versions[i] = indexRequest.version();
// update the version on request so it will happen on the replicas
indexRequest.version(version);
successSize++;
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < i; j++) {
applyVersion(request.items().get(j), versions[j]);
}
logger.error(e.getMessage(), e);
throw new ElasticsearchException(e.getMessage());
}
if (e instanceof ElasticsearchException
&& ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
logger.error(
"[{}][{}] failed to execute bulk item (index) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
indexRequest);
} else {
logger.error(
"[{}][{}] failed to execute bulk item (index) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
indexRequest);
}
failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
// nullify the request so it won't execute on the replicas
request.items().set(i, null);
} finally {
// update mapping on master if needed, we won't update changes to the same type, since
// once its changed, it won't have mappers added
if (op != null && op.parsedDoc().mappingsModified()) {
mappingsToUpdate.add(Tuple.tuple(indexRequest.index(), indexRequest.type()));
}
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
try {
Engine.Delete delete =
indexShard
.prepareDelete(deleteRequest.type(), deleteRequest.id(), deleteRequest.version())
.versionType(deleteRequest.versionType())
.origin(Engine.Operation.Origin.PRIMARY);
indexShard.delete(delete);
// update the request with teh version so it will go to the replicas
deleteRequest.version(delete.version());
successSize++;
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < i; j++) {
applyVersion(request.items().get(j), versions[j]);
}
logger.error(e.getMessage(), e);
throw new ElasticsearchException(e.getMessage());
}
if (e instanceof ElasticsearchException
&& ((ElasticsearchException) e).status() == RestStatus.CONFLICT) {
logger.trace(
"[{}][{}] failed to execute bulk item (delete) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
deleteRequest);
} else {
logger.debug(
"[{}][{}] failed to execute bulk item (delete) {}",
e,
shardRequest.request.index(),
shardRequest.shardId,
deleteRequest);
}
failure.add(new IngestItemFailure(item.id(), ExceptionsHelper.detailedMessage(e)));
// nullify the request so it won't execute on the replicas
request.items().set(i, null);
}
}
}
if (!mappingsToUpdate.isEmpty()) {
for (Tuple<String, String> mappingToUpdate : mappingsToUpdate) {
logger.info("mapping update {} {}", mappingToUpdate.v1(), mappingToUpdate.v2());
updateMappingOnMaster(mappingToUpdate.v1(), mappingToUpdate.v2());
}
}
IngestShardResponse response =
new IngestShardResponse(
new ShardId(request.index(), request.shardId()), successSize, failure);
return new PrimaryResponse<IngestShardResponse, IngestShardRequest>(
shardRequest.request, response, null);
}
private void executeBulk(
final BulkRequest bulkRequest,
final long startTime,
final ActionListener<BulkResponse> listener,
final AtomicArray<BulkItemResponse> responses) {
final ClusterState clusterState = clusterService.state();
// TODO use timeout to wait here if its blocked...
clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);
final ConcreteIndices concreteIndices =
new ConcreteIndices(clusterState, indexNameExpressionResolver);
MetaData metaData = clusterState.metaData();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof DocumentRequest) {
DocumentRequest req = (DocumentRequest) request;
if (addFailureIfIndexIsUnavailable(
req, bulkRequest, responses, i, concreteIndices, metaData)) {
continue;
}
String concreteIndex = concreteIndices.resolveIfAbsent(req);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
MappingMetaData mappingMd = null;
if (metaData.hasIndex(concreteIndex)) {
mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type());
}
try {
indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
} catch (ElasticsearchParseException | RoutingMissingException e) {
BulkItemResponse.Failure failure =
new BulkItemResponse.Failure(
concreteIndex, indexRequest.type(), indexRequest.id(), e);
BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
responses.set(i, bulkItemResponse);
// make sure the request gets never processed again
bulkRequest.requests.set(i, null);
}
} else {
concreteIndices.resolveIfAbsent(req);
req.routing(
clusterState
.metaData()
.resolveIndexRouting(req.parent(), req.routing(), req.index()));
}
}
}
// first, go over all the requests and create a ShardId -> Operations mapping
Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index());
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
concreteIndex,
indexRequest.type(),
indexRequest.id(),
indexRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index());
MappingMetaData mappingMd =
clusterState.metaData().index(concreteIndex).mappingOrDefault(deleteRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& deleteRequest.routing() == null) {
// if routing is required, and no routing on the delete request, we need to broadcast
// it....
GroupShardsIterator groupShards =
clusterService.operationRouting().broadcastDeleteShards(clusterState, concreteIndex);
for (ShardIterator shardIt : groupShards) {
List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardIt.shardId(), list);
}
list.add(new BulkItemRequest(i, deleteRequest));
}
} else {
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
concreteIndex,
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
} else if (request instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) request;
String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index());
MappingMetaData mappingMd =
clusterState.metaData().index(concreteIndex).mappingOrDefault(updateRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& updateRequest.routing() == null) {
BulkItemResponse.Failure failure =
new BulkItemResponse.Failure(
updateRequest.index(),
updateRequest.type(),
updateRequest.id(),
new IllegalArgumentException("routing is required for this item"));
responses.set(i, new BulkItemResponse(i, updateRequest.type(), failure));
continue;
}
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
concreteIndex,
updateRequest.type(),
updateRequest.id(),
updateRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
}
if (requestsByShard.isEmpty()) {
listener.onResponse(
new BulkResponse(
responses.toArray(new BulkItemResponse[responses.length()]),
buildTookInMillis(startTime)));
return;
}
final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
final ShardId shardId = entry.getKey();
final List<BulkItemRequest> requests = entry.getValue();
BulkShardRequest bulkShardRequest =
new BulkShardRequest(
bulkRequest,
shardId,
bulkRequest.refresh(),
requests.toArray(new BulkItemRequest[requests.size()]));
bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
bulkShardRequest.timeout(bulkRequest.timeout());
shardBulkAction.execute(
bulkShardRequest,
new ActionListener<BulkShardResponse>() {
@Override
public void onResponse(BulkShardResponse bulkShardResponse) {
for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
// we may have no response if item failed
if (bulkItemResponse.getResponse() != null) {
bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
}
responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
@Override
public void onFailure(Throwable e) {
// create failures for all relevant requests
for (BulkItemRequest request : requests) {
if (request.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request.request();
responses.set(
request.id(),
new BulkItemResponse(
request.id(),
indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
new BulkItemResponse.Failure(
concreteIndices.getConcreteIndex(indexRequest.index()),
indexRequest.type(),
indexRequest.id(),
e)));
} else if (request.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request.request();
responses.set(
request.id(),
new BulkItemResponse(
request.id(),
"delete",
new BulkItemResponse.Failure(
concreteIndices.getConcreteIndex(deleteRequest.index()),
deleteRequest.type(),
deleteRequest.id(),
e)));
} else if (request.request() instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) request.request();
responses.set(
request.id(),
new BulkItemResponse(
request.id(),
"update",
new BulkItemResponse.Failure(
concreteIndices.getConcreteIndex(updateRequest.index()),
updateRequest.type(),
updateRequest.id(),
e)));
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
private void finishHim() {
listener.onResponse(
new BulkResponse(
responses.toArray(new BulkItemResponse[responses.length()]),
buildTookInMillis(startTime)));
}
});
}
}
@Test
public void testMappingMetaDataParsed() throws Exception {
logger.info("--> starting 1 nodes");
internalCluster().startNode(settingsBuilder().put("gateway.type", "local"));
logger.info("--> creating test index, with meta routing");
client()
.admin()
.indices()
.prepareCreate("test")
.addMapping(
"type1",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("_routing")
.field("required", true)
.endObject()
.endObject()
.endObject())
.execute()
.actionGet();
logger.info("--> waiting for yellow status");
ensureYellow();
logger.info("--> verify meta _routing required exists");
MappingMetaData mappingMd =
client()
.admin()
.cluster()
.prepareState()
.execute()
.actionGet()
.getState()
.metaData()
.index("test")
.mapping("type1");
assertThat(mappingMd.routing().required(), equalTo(true));
logger.info("--> restarting nodes...");
internalCluster().fullRestart();
logger.info("--> waiting for yellow status");
ensureYellow();
logger.info("--> verify meta _routing required exists");
mappingMd =
client()
.admin()
.cluster()
.prepareState()
.execute()
.actionGet()
.getState()
.metaData()
.index("test")
.mapping("type1");
assertThat(mappingMd.routing().required(), equalTo(true));
}
private void executeBulk(
final BulkRequest bulkRequest,
final long startTime,
final ActionListener<BulkResponse> listener) {
ClusterState clusterState = clusterService.state();
for (ActionRequest request : bulkRequest.requests) {
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
indexRequest.routing(
clusterState
.metaData()
.resolveIndexRouting(indexRequest.routing(), indexRequest.index()));
indexRequest.index(clusterState.metaData().concreteIndex(indexRequest.index()));
if (allowIdGeneration) {
if (indexRequest.id() == null) {
indexRequest.id(UUID.randomBase64UUID());
// since we generate the id, change it to CREATE
indexRequest.opType(IndexRequest.OpType.CREATE);
}
}
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
deleteRequest.index(clusterState.metaData().concreteIndex(deleteRequest.index()));
}
}
final BulkItemResponse[] responses = new BulkItemResponse[bulkRequest.requests.size()];
// first, go over all the requests and create a ShardId -> Operations mapping
Map<ShardId, List<BulkItemRequest>> requestsByShard = Maps.newHashMap();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
// handle routing
MappingMetaData mappingMd =
clusterState.metaData().index(indexRequest.index()).mapping(indexRequest.type());
if (mappingMd != null) {
try {
indexRequest.processRouting(mappingMd);
} catch (ElasticSearchException e) {
responses[i] =
new BulkItemResponse(
i,
indexRequest.opType().toString().toLowerCase(),
new BulkItemResponse.Failure(
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
e.getDetailedMessage()));
continue;
}
}
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
indexRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
MappingMetaData mappingMd =
clusterState.metaData().index(deleteRequest.index()).mapping(deleteRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& deleteRequest.routing() == null) {
// if routing is required, and no routing on the delete request, we need to broadcast
// it....
GroupShardsIterator groupShards =
clusterService
.operationRouting()
.broadcastDeleteShards(clusterState, deleteRequest.index());
for (ShardIterator shardIt : groupShards) {
List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardIt.shardId(), list);
}
list.add(new BulkItemRequest(i, request));
}
} else {
ShardId shardId =
clusterService
.operationRouting()
.deleteShards(
clusterState,
deleteRequest.index(),
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = Lists.newArrayList();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
}
}
if (requestsByShard.isEmpty()) {
listener.onResponse(new BulkResponse(responses, System.currentTimeMillis() - startTime));
return;
}
final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
final ShardId shardId = entry.getKey();
final List<BulkItemRequest> requests = entry.getValue();
BulkShardRequest bulkShardRequest =
new BulkShardRequest(
shardId.index().name(),
shardId.id(),
bulkRequest.refresh(),
requests.toArray(new BulkItemRequest[requests.size()]));
bulkShardRequest.replicationType(bulkRequest.replicationType());
bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
shardBulkAction.execute(
bulkShardRequest,
new ActionListener<BulkShardResponse>() {
@Override
public void onResponse(BulkShardResponse bulkShardResponse) {
synchronized (responses) {
for (BulkItemResponse bulkItemResponse : bulkShardResponse.responses()) {
responses[bulkItemResponse.itemId()] = bulkItemResponse;
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
@Override
public void onFailure(Throwable e) {
// create failures for all relevant requests
String message = ExceptionsHelper.detailedMessage(e);
synchronized (responses) {
for (BulkItemRequest request : requests) {
if (request.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request.request();
responses[request.id()] =
new BulkItemResponse(
request.id(),
indexRequest.opType().toString().toLowerCase(),
new BulkItemResponse.Failure(
indexRequest.index(),
indexRequest.type(),
indexRequest.id(),
message));
} else if (request.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request.request();
responses[request.id()] =
new BulkItemResponse(
request.id(),
"delete",
new BulkItemResponse.Failure(
deleteRequest.index(),
deleteRequest.type(),
deleteRequest.id(),
message));
}
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
private void finishHim() {
listener.onResponse(
new BulkResponse(responses, System.currentTimeMillis() - startTime));
}
});
}
}