@Override
public List<ActionRequest> doCall(Index index) {
List<ActionRequest> inserts = new ArrayList<ActionRequest>();
List<UpdateRequest> updates = index.getNormalizer().normalize(dto);
for (UpdateRequest update : updates) {
if (update.doc() != null) {
inserts.add(
update
.upsertRequest()
.index(index.getIndexName())
.type(index.getIndexType())
.id(update.id())
.routing(update.routing()));
} else {
inserts.add(update.index(index.getIndexName()).type(index.getIndexType()));
}
}
return inserts;
}
@Override
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client)
throws IOException {
UpdateRequest updateRequest =
new UpdateRequest(request.param("index"), request.param("type"), request.param("id"));
updateRequest.routing(request.param("routing"));
updateRequest.parent(request.param("parent"));
updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout()));
updateRequest.setRefreshPolicy(request.param("refresh"));
String waitForActiveShards = request.param("wait_for_active_shards");
if (waitForActiveShards != null) {
updateRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards));
}
updateRequest.docAsUpsert(request.paramAsBoolean("doc_as_upsert", updateRequest.docAsUpsert()));
FetchSourceContext fetchSourceContext = FetchSourceContext.parseFromRestRequest(request);
String sField = request.param("fields");
if (sField != null && fetchSourceContext != null) {
throw new IllegalArgumentException(
"[fields] and [_source] cannot be used in the same request");
}
if (sField != null) {
DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead");
String[] sFields = Strings.splitStringByCommaToArray(sField);
updateRequest.fields(sFields);
} else if (fetchSourceContext != null) {
updateRequest.fetchSource(fetchSourceContext);
}
updateRequest.retryOnConflict(
request.paramAsInt("retry_on_conflict", updateRequest.retryOnConflict()));
updateRequest.version(RestActions.parseVersion(request));
updateRequest.versionType(
VersionType.fromString(request.param("version_type"), updateRequest.versionType()));
// see if we have it in the body
if (request.hasContent()) {
updateRequest.fromXContent(request.content());
IndexRequest upsertRequest = updateRequest.upsertRequest();
if (upsertRequest != null) {
upsertRequest.routing(request.param("routing"));
upsertRequest.parent(
request.param(
"parent")); // order is important, set it after routing, so it will set the routing
upsertRequest.timestamp(request.param("timestamp"));
if (request.hasParam("ttl")) {
upsertRequest.ttl(request.param("ttl"));
}
upsertRequest.version(RestActions.parseVersion(request));
upsertRequest.versionType(
VersionType.fromString(request.param("version_type"), upsertRequest.versionType()));
}
IndexRequest doc = updateRequest.doc();
if (doc != null) {
doc.routing(request.param("routing"));
doc.parent(
request.param(
"parent")); // order is important, set it after routing, so it will set the routing
doc.timestamp(request.param("timestamp"));
if (request.hasParam("ttl")) {
doc.ttl(request.param("ttl"));
}
doc.version(RestActions.parseVersion(request));
doc.versionType(VersionType.fromString(request.param("version_type"), doc.versionType()));
}
}
return channel ->
client.update(
updateRequest,
new RestStatusToXContentListener<>(
channel, r -> r.getLocation(updateRequest.routing())));
}
@Override
public void handleRequest(final RestRequest request, final RestChannel channel) {
UpdateRequest updateRequest =
new UpdateRequest(request.param("index"), request.param("type"), request.param("id"));
updateRequest.listenerThreaded(false);
updateRequest.routing(request.param("routing"));
updateRequest.parent(
request.param(
"parent")); // order is important, set it after routing, so it will set the routing
updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout()));
updateRequest.refresh(request.paramAsBoolean("refresh", updateRequest.refresh()));
String replicationType = request.param("replication");
if (replicationType != null) {
updateRequest.replicationType(ReplicationType.fromString(replicationType));
}
String consistencyLevel = request.param("consistency");
if (consistencyLevel != null) {
updateRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel));
}
updateRequest.percolate(request.param("percolate", null));
updateRequest.script(request.param("script"));
updateRequest.scriptLang(request.param("lang"));
for (Map.Entry<String, String> entry : request.params().entrySet()) {
if (entry.getKey().startsWith("sp_")) {
updateRequest.addScriptParam(entry.getKey().substring(3), entry.getValue());
}
}
String sField = request.param("fields");
if (sField != null) {
String[] sFields = Strings.splitStringByCommaToArray(sField);
if (sFields != null) {
updateRequest.fields(sFields);
}
}
updateRequest.retryOnConflict(
request.paramAsInt("retry_on_conflict", updateRequest.retryOnConflict()));
// see if we have it in the body
if (request.hasContent()) {
try {
updateRequest.source(
request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength());
IndexRequest upsertRequest = updateRequest.upsertRequest();
if (upsertRequest != null) {
upsertRequest.routing(request.param("routing"));
upsertRequest.parent(
request.param(
"parent")); // order is important, set it after routing, so it will set the
// routing
upsertRequest.timestamp(request.param("timestamp"));
if (request.hasParam("ttl")) {
upsertRequest.ttl(request.paramAsTime("ttl", null).millis());
}
upsertRequest.version(RestActions.parseVersion(request));
upsertRequest.versionType(
VersionType.fromString(request.param("version_type"), upsertRequest.versionType()));
}
IndexRequest doc = updateRequest.doc();
if (doc != null) {
doc.routing(request.param("routing"));
doc.parent(
request.param(
"parent")); // order is important, set it after routing, so it will set the
// routing
doc.timestamp(request.param("timestamp"));
if (request.hasParam("ttl")) {
doc.ttl(request.paramAsTime("ttl", null).millis());
}
doc.version(RestActions.parseVersion(request));
doc.versionType(VersionType.fromString(request.param("version_type"), doc.versionType()));
}
} catch (Exception e) {
try {
channel.sendResponse(new XContentThrowableRestResponse(request, e));
} catch (IOException e1) {
logger.warn("Failed to send response", e1);
}
return;
}
}
client.update(
updateRequest,
new ActionListener<UpdateResponse>() {
@Override
public void onResponse(UpdateResponse response) {
try {
XContentBuilder builder = RestXContentBuilder.restContentBuilder(request);
builder
.startObject()
.field(Fields.OK, true)
.field(Fields._INDEX, response.index())
.field(Fields._TYPE, response.type())
.field(Fields._ID, response.id())
.field(Fields._VERSION, response.version());
if (response.getResult() != null) {
builder.startObject(Fields.GET);
response.getResult().toXContentEmbedded(builder, request);
builder.endObject();
}
if (response.matches() != null) {
builder.startArray(Fields.MATCHES);
for (String match : response.matches()) {
builder.value(match);
}
builder.endArray();
}
builder.endObject();
RestStatus status = OK;
if (response.version() == 1) {
status = CREATED;
}
channel.sendResponse(new XContentRestResponse(request, status, builder));
} catch (Exception e) {
onFailure(e);
}
}
@Override
public void onFailure(Throwable e) {
try {
channel.sendResponse(new XContentThrowableRestResponse(request, e));
} catch (IOException e1) {
logger.error("Failed to send failure response", e1);
}
}
});
}
private void executeBulk(
final BulkRequest bulkRequest,
final long startTime,
final ActionListener<BulkResponse> listener,
final AtomicArray<BulkItemResponse> responses) {
final ClusterState clusterState = clusterService.state();
// TODO use timeout to wait here if its blocked...
clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);
final ConcreteIndices concreteIndices =
new ConcreteIndices(clusterState, indexNameExpressionResolver);
MetaData metaData = clusterState.metaData();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof DocumentRequest) {
DocumentRequest req = (DocumentRequest) request;
if (addFailureIfIndexIsUnavailable(
req, bulkRequest, responses, i, concreteIndices, metaData)) {
continue;
}
String concreteIndex = concreteIndices.resolveIfAbsent(req);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
MappingMetaData mappingMd = null;
if (metaData.hasIndex(concreteIndex)) {
mappingMd = metaData.index(concreteIndex).mappingOrDefault(indexRequest.type());
}
try {
indexRequest.process(metaData, mappingMd, allowIdGeneration, concreteIndex);
} catch (ElasticsearchParseException | RoutingMissingException e) {
BulkItemResponse.Failure failure =
new BulkItemResponse.Failure(
concreteIndex, indexRequest.type(), indexRequest.id(), e);
BulkItemResponse bulkItemResponse = new BulkItemResponse(i, "index", failure);
responses.set(i, bulkItemResponse);
// make sure the request gets never processed again
bulkRequest.requests.set(i, null);
}
} else {
concreteIndices.resolveIfAbsent(req);
req.routing(
clusterState
.metaData()
.resolveIndexRouting(req.parent(), req.routing(), req.index()));
}
}
}
// first, go over all the requests and create a ShardId -> Operations mapping
Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>();
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index());
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
concreteIndex,
indexRequest.type(),
indexRequest.id(),
indexRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index());
MappingMetaData mappingMd =
clusterState.metaData().index(concreteIndex).mappingOrDefault(deleteRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& deleteRequest.routing() == null) {
// if routing is required, and no routing on the delete request, we need to broadcast
// it....
GroupShardsIterator groupShards =
clusterService.operationRouting().broadcastDeleteShards(clusterState, concreteIndex);
for (ShardIterator shardIt : groupShards) {
List<BulkItemRequest> list = requestsByShard.get(shardIt.shardId());
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardIt.shardId(), list);
}
list.add(new BulkItemRequest(i, deleteRequest));
}
} else {
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
concreteIndex,
deleteRequest.type(),
deleteRequest.id(),
deleteRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
} else if (request instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) request;
String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index());
MappingMetaData mappingMd =
clusterState.metaData().index(concreteIndex).mappingOrDefault(updateRequest.type());
if (mappingMd != null
&& mappingMd.routing().required()
&& updateRequest.routing() == null) {
BulkItemResponse.Failure failure =
new BulkItemResponse.Failure(
updateRequest.index(),
updateRequest.type(),
updateRequest.id(),
new IllegalArgumentException("routing is required for this item"));
responses.set(i, new BulkItemResponse(i, updateRequest.type(), failure));
continue;
}
ShardId shardId =
clusterService
.operationRouting()
.indexShards(
clusterState,
concreteIndex,
updateRequest.type(),
updateRequest.id(),
updateRequest.routing())
.shardId();
List<BulkItemRequest> list = requestsByShard.get(shardId);
if (list == null) {
list = new ArrayList<>();
requestsByShard.put(shardId, list);
}
list.add(new BulkItemRequest(i, request));
}
}
if (requestsByShard.isEmpty()) {
listener.onResponse(
new BulkResponse(
responses.toArray(new BulkItemResponse[responses.length()]),
buildTookInMillis(startTime)));
return;
}
final AtomicInteger counter = new AtomicInteger(requestsByShard.size());
for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
final ShardId shardId = entry.getKey();
final List<BulkItemRequest> requests = entry.getValue();
BulkShardRequest bulkShardRequest =
new BulkShardRequest(
bulkRequest,
shardId,
bulkRequest.refresh(),
requests.toArray(new BulkItemRequest[requests.size()]));
bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
bulkShardRequest.timeout(bulkRequest.timeout());
shardBulkAction.execute(
bulkShardRequest,
new ActionListener<BulkShardResponse>() {
@Override
public void onResponse(BulkShardResponse bulkShardResponse) {
for (BulkItemResponse bulkItemResponse : bulkShardResponse.getResponses()) {
// we may have no response if item failed
if (bulkItemResponse.getResponse() != null) {
bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo());
}
responses.set(bulkItemResponse.getItemId(), bulkItemResponse);
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
@Override
public void onFailure(Throwable e) {
// create failures for all relevant requests
for (BulkItemRequest request : requests) {
if (request.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request.request();
responses.set(
request.id(),
new BulkItemResponse(
request.id(),
indexRequest.opType().toString().toLowerCase(Locale.ENGLISH),
new BulkItemResponse.Failure(
concreteIndices.getConcreteIndex(indexRequest.index()),
indexRequest.type(),
indexRequest.id(),
e)));
} else if (request.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request.request();
responses.set(
request.id(),
new BulkItemResponse(
request.id(),
"delete",
new BulkItemResponse.Failure(
concreteIndices.getConcreteIndex(deleteRequest.index()),
deleteRequest.type(),
deleteRequest.id(),
e)));
} else if (request.request() instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) request.request();
responses.set(
request.id(),
new BulkItemResponse(
request.id(),
"update",
new BulkItemResponse.Failure(
concreteIndices.getConcreteIndex(updateRequest.index()),
updateRequest.type(),
updateRequest.id(),
e)));
}
}
if (counter.decrementAndGet() == 0) {
finishHim();
}
}
private void finishHim() {
listener.onResponse(
new BulkResponse(
responses.toArray(new BulkItemResponse[responses.length()]),
buildTookInMillis(startTime)));
}
});
}
}