@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeLong(tookInMillis);
out.writeLong(ingestId);
shardId.writeTo(out);
out.writeVInt(successCount);
out.writeVInt(quorumShards);
out.writeVInt(actionRequests.size());
for (ActionRequest actionRequest : actionRequests) {
if (actionRequest == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
if (actionRequest instanceof IndexRequest) {
out.writeBoolean(true);
} else if (actionRequest instanceof DeleteRequest) {
out.writeBoolean(false);
} else {
throw new ElasticsearchIllegalStateException(
"action request not supported: " + actionRequest.getClass().getName());
}
actionRequest.writeTo(out);
}
}
out.writeVInt(failures.size());
for (IngestActionFailure f : failures) {
f.writeTo(out);
}
}
private static void copyHeadersAndContext(
ActionRequest actionRequest, RestRequest restRequest, Set<String> headers) {
for (String usefulHeader : headers) {
String headerValue = restRequest.header(usefulHeader);
if (headerValue != null) {
actionRequest.putHeader(usefulHeader, headerValue);
}
}
actionRequest.copyContextFrom(restRequest);
}
private static void assertHeaders(ActionRequest<?> request, Map<String, String> headers) {
if (headers.size() == 0) {
assertThat(request.getHeaders() == null || request.getHeaders().size() == 0, equalTo(true));
} else {
assertThat(request.getHeaders(), notNullValue());
assertThat(request.getHeaders().size(), equalTo(headers.size()));
for (String key : request.getHeaders()) {
assertThat(headers.get(key), equalTo(request.getHeader(key)));
}
}
}
private static void assertContext(ActionRequest<?> request, Map<String, String> context) {
if (context.size() == 0) {
assertThat(request.isContextEmpty(), is(true));
} else {
ImmutableOpenMap map = request.getContext();
assertThat(map, notNullValue());
assertThat(map.size(), equalTo(context.size()));
for (Object key : map.keys()) {
assertThat(context.get(key), equalTo(request.getFromContext(key)));
}
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(index);
out.writeInt(shardId);
timeout.writeTo(out);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeVInt(requests.size());
for (SearchRequest request : requests) {
request.writeTo(out);
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeByte(replicationType.id());
out.writeByte(consistencyLevel.id());
timeout.writeTo(out);
out.writeString(index);
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
replicationType = ReplicationType.fromId(in.readByte());
consistencyLevel = WriteConsistencyLevel.fromId(in.readByte());
timeout = TimeValue.readTimeValue(in);
index = in.readString();
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
sourceNodeId = in.readString();
transferId = new UUID(in.readLong(), in.readLong());
currentPos = in.readVInt();
content = in.readBytesReference();
isLast = in.readBoolean();
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
index = in.readString();
shardId = in.readInt();
timeout = TimeValue.readTimeValue(in);
// no need to pass threading over the network, they are always false when coming throw a thread
// pool
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeByte(replicationType.id());
out.writeByte(consistencyLevel.id());
timeout.writeTo(out);
out.writeStringArrayNullable(indices);
indicesOptions.writeIndicesOptions(out);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
taskId.writeTo(out);
parentTaskId.writeTo(out);
out.writeStringArrayNullable(nodesIds);
out.writeStringArrayNullable(actions);
out.writeOptionalStreamable(timeout);
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
int size = in.readVInt();
for (int i = 0; i < size; i++) {
SearchRequest request = new SearchRequest();
request.readFrom(in);
requests.add(request);
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(sourceNodeId);
out.writeLong(transferId.getMostSignificantBits());
out.writeLong(transferId.getLeastSignificantBits());
out.writeVLong(currentPos);
out.writeBytesReference(content);
out.writeBoolean(isLast);
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
taskId = TaskId.readFromStream(in);
parentTaskId = TaskId.readFromStream(in);
nodesIds = in.readStringArray();
actions = in.readStringArray();
if (in.readBoolean()) {
timeout = TimeValue.readTimeValue(in);
}
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
if (in.readBoolean()) {
shardId = ShardId.readShardId(in);
} else {
shardId = null;
}
consistencyLevel = WriteConsistencyLevel.fromId(in.readByte());
timeout = TimeValue.readTimeValue(in);
index = in.readString();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeOptionalString(preference);
out.writeBoolean(refresh);
out.writeBoolean(realtime);
out.writeVInt(items.size());
for (Item item : items) {
item.writeTo(out);
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
if (shardId != null) {
out.writeBoolean(true);
shardId.writeTo(out);
} else {
out.writeBoolean(false);
}
out.writeByte(consistencyLevel.id());
timeout.writeTo(out);
out.writeString(index);
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
preference = in.readOptionalString();
refresh = in.readBoolean();
realtime = in.readBoolean();
int size = in.readVInt();
items = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
items.add(Item.readItem(in));
}
}
private boolean setResponseFailureIfIndexMatches(
AtomicArray<BulkItemResponse> responses,
int idx,
ActionRequest request,
String index,
Throwable e) {
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
if (index.equals(indexRequest.index())) {
responses.set(
idx,
new BulkItemResponse(
idx,
"index",
new BulkItemResponse.Failure(
indexRequest.index(), indexRequest.type(), indexRequest.id(), e)));
return true;
}
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
if (index.equals(deleteRequest.index())) {
responses.set(
idx,
new BulkItemResponse(
idx,
"delete",
new BulkItemResponse.Failure(
deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), e)));
return true;
}
} else if (request instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) request;
if (index.equals(updateRequest.index())) {
responses.set(
idx,
new BulkItemResponse(
idx,
"update",
new BulkItemResponse.Failure(
updateRequest.index(), updateRequest.type(), updateRequest.id(), e)));
return true;
}
} else {
throw new ElasticsearchException(
"Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
}
return false;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeOptionalString(preference);
out.writeBoolean(refresh);
if (realtime == null) {
out.writeByte((byte) -1);
} else if (realtime == false) {
out.writeByte((byte) 0);
} else {
out.writeByte((byte) 1);
}
out.writeVInt(items.size());
for (Item item : items) {
item.writeTo(out);
}
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
preference = in.readOptionalString();
refresh = in.readBoolean();
byte realtime = in.readByte();
if (realtime == 0) {
this.realtime = false;
} else if (realtime == 1) {
this.realtime = true;
}
ignoreErrorsOnGeneratedFields = in.readBoolean();
int size = in.readVInt();
items = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
items.add(Item.readItem(in));
}
}
private void assertRequestContainsHeader(ActionRequest request, Map<String, String> context) {
String msg =
String.format(
Locale.ROOT,
"Expected header %s to be in request %s",
randomHeaderKey,
request.getClass().getName());
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
msg =
String.format(
Locale.ROOT,
"Expected header %s to be in index request %s/%s/%s",
randomHeaderKey,
indexRequest.index(),
indexRequest.type(),
indexRequest.id());
}
assertThat(msg, context.containsKey(randomHeaderKey), is(true));
assertThat(context.get(randomHeaderKey).toString(), is(randomHeaderValue));
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
requests = in.readStreamableList(SearchTemplateRequest::new);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeStreamableList(requests);
}
@Override
protected void doExecute(
final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
final long startTime = System.currentTimeMillis();
final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
if (autoCreateIndex.needToCheck()) {
// Keep track of all unique indices and all unique types per index for the create index
// requests:
final Map<String, Set<String>> indicesAndTypes = new HashMap<>();
for (ActionRequest request : bulkRequest.requests) {
if (request instanceof DocumentRequest) {
DocumentRequest req = (DocumentRequest) request;
Set<String> types = indicesAndTypes.get(req.index());
if (types == null) {
indicesAndTypes.put(req.index(), types = new HashSet<>());
}
types.add(req.type());
} else {
throw new ElasticsearchException(
"Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
}
}
final AtomicInteger counter = new AtomicInteger(indicesAndTypes.size());
ClusterState state = clusterService.state();
for (Map.Entry<String, Set<String>> entry : indicesAndTypes.entrySet()) {
final String index = entry.getKey();
if (autoCreateIndex.shouldAutoCreate(index, state)) {
CreateIndexRequest createIndexRequest = new CreateIndexRequest();
createIndexRequest.index(index);
for (String type : entry.getValue()) {
createIndexRequest.mapping(type);
}
createIndexRequest.cause("auto(bulk api)");
createIndexRequest.masterNodeTimeout(bulkRequest.timeout());
createIndexAction.execute(
createIndexRequest,
new ActionListener<CreateIndexResponse>() {
@Override
public void onResponse(CreateIndexResponse result) {
if (counter.decrementAndGet() == 0) {
try {
executeBulk(bulkRequest, startTime, listener, responses);
} catch (Throwable t) {
listener.onFailure(t);
}
}
}
@Override
public void onFailure(Throwable e) {
if (!(ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException)) {
// fail all requests involving this index, if create didnt work
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request != null
&& setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
bulkRequest.requests.set(i, null);
}
}
}
if (counter.decrementAndGet() == 0) {
try {
executeBulk(bulkRequest, startTime, listener, responses);
} catch (Throwable t) {
listener.onFailure(t);
}
}
}
});
} else {
if (counter.decrementAndGet() == 0) {
executeBulk(bulkRequest, startTime, listener, responses);
}
}
}
} else {
executeBulk(bulkRequest, startTime, listener, responses);
}
}
private static void putContext(ActionRequest<?> request, Map<String, String> context) {
for (Map.Entry<String, String> header : context.entrySet()) {
request.putInContext(header.getKey(), header.getValue());
}
}
private static void putHeaders(ActionRequest<?> request, Map<String, String> headers) {
for (Map.Entry<String, String> header : headers.entrySet()) {
request.putHeader(header.getKey(), header.getValue());
}
}
@Override
protected void doExecute(ActionRequest request, ActionListener listener) {
listener.onFailure(new InternalException(actionName, request.getHeaders()));
}