@Override
protected void doExecute(
final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
final long startTime = System.currentTimeMillis();
Set<String> indices = Sets.newHashSet();
for (ActionRequest request : bulkRequest.requests) {
if (request instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) request;
if (!indices.contains(indexRequest.index())) {
indices.add(indexRequest.index());
}
} else if (request instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) request;
if (!indices.contains(deleteRequest.index())) {
indices.add(deleteRequest.index());
}
}
}
if (autoCreateIndex) {
final AtomicInteger counter = new AtomicInteger(indices.size());
final AtomicBoolean failed = new AtomicBoolean();
for (String index : indices) {
if (!clusterService.state().metaData().hasConcreteIndex(index)) {
createIndexAction.execute(
new CreateIndexRequest(index).cause("auto(bulk api)"),
new ActionListener<CreateIndexResponse>() {
@Override
public void onResponse(CreateIndexResponse result) {
if (counter.decrementAndGet() == 0) {
executeBulk(bulkRequest, startTime, listener);
}
}
@Override
public void onFailure(Throwable e) {
if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
// we have the index, do it
if (counter.decrementAndGet() == 0) {
executeBulk(bulkRequest, startTime, listener);
}
} else if (failed.compareAndSet(false, true)) {
listener.onFailure(e);
}
}
});
} else {
if (counter.decrementAndGet() == 0) {
executeBulk(bulkRequest, startTime, listener);
}
}
}
} else {
executeBulk(bulkRequest, startTime, listener);
}
}
/**
* This method executes the {@link BulkRequest} and calls the given listener once the request
* returns. This method will not create any indices even if auto-create indices is enabled.
*
* @see #doExecute(BulkRequest, org.elasticsearch.action.ActionListener)
*/
public void executeBulk(
final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
final long startTime = System.currentTimeMillis();
executeBulk(
bulkRequest,
startTime,
listener,
new AtomicArray<BulkItemResponse>(bulkRequest.requests.size()));
}
private BulkRequest processBulkIfNeeded(BulkRequest bulkRequest, boolean force) {
if ((force && bulkRequest.numberOfActions() > 0) || bulkRequest.numberOfActions() >= bulkSize) {
try {
bulkAction.executeBulk(
bulkRequest,
new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse bulkResponse) {
if (bulkResponse.hasFailures()) {
int failedItems = 0;
for (BulkItemResponse response : bulkResponse) {
if (response.isFailed()) failedItems++;
}
if (logger.isTraceEnabled()) {
logger.trace(
"bulk deletion failures for [{}]/[{}] items, failure message: [{}]",
failedItems,
bulkResponse.getItems().length,
bulkResponse.buildFailureMessage());
} else {
logger.error(
"bulk deletion failures for [{}]/[{}] items",
failedItems,
bulkResponse.getItems().length);
}
} else {
logger.trace("bulk deletion took {}ms", bulkResponse.getTookInMillis());
}
}
@Override
public void onFailure(Throwable e) {
if (logger.isTraceEnabled()) {
logger.trace("failed to execute bulk", e);
} else {
logger.warn("failed to execute bulk: ", e);
}
}
});
} catch (Exception e) {
logger.warn("failed to process bulk", e);
}
bulkRequest = new BulkRequest();
}
return bulkRequest;
}
@Override
protected void doExecute(
final BulkRequest bulkRequest, final ActionListener<BulkResponse> listener) {
final long startTime = System.currentTimeMillis();
final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
if (autoCreateIndex.needToCheck()) {
// Keep track of all unique indices and all unique types per index for the create index
// requests:
final Map<String, Set<String>> indicesAndTypes = new HashMap<>();
for (ActionRequest request : bulkRequest.requests) {
if (request instanceof DocumentRequest) {
DocumentRequest req = (DocumentRequest) request;
Set<String> types = indicesAndTypes.get(req.index());
if (types == null) {
indicesAndTypes.put(req.index(), types = new HashSet<>());
}
types.add(req.type());
} else {
throw new ElasticsearchException(
"Parsed unknown request in bulk actions: " + request.getClass().getSimpleName());
}
}
final AtomicInteger counter = new AtomicInteger(indicesAndTypes.size());
ClusterState state = clusterService.state();
for (Map.Entry<String, Set<String>> entry : indicesAndTypes.entrySet()) {
final String index = entry.getKey();
if (autoCreateIndex.shouldAutoCreate(index, state)) {
CreateIndexRequest createIndexRequest = new CreateIndexRequest();
createIndexRequest.index(index);
for (String type : entry.getValue()) {
createIndexRequest.mapping(type);
}
createIndexRequest.cause("auto(bulk api)");
createIndexRequest.masterNodeTimeout(bulkRequest.timeout());
createIndexAction.execute(
createIndexRequest,
new ActionListener<CreateIndexResponse>() {
@Override
public void onResponse(CreateIndexResponse result) {
if (counter.decrementAndGet() == 0) {
try {
executeBulk(bulkRequest, startTime, listener, responses);
} catch (Throwable t) {
listener.onFailure(t);
}
}
}
@Override
public void onFailure(Throwable e) {
if (!(ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException)) {
// fail all requests involving this index, if create didnt work
for (int i = 0; i < bulkRequest.requests.size(); i++) {
ActionRequest request = bulkRequest.requests.get(i);
if (request != null
&& setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
bulkRequest.requests.set(i, null);
}
}
}
if (counter.decrementAndGet() == 0) {
try {
executeBulk(bulkRequest, startTime, listener, responses);
} catch (Throwable t) {
listener.onFailure(t);
}
}
}
});
} else {
if (counter.decrementAndGet() == 0) {
executeBulk(bulkRequest, startTime, listener, responses);
}
}
}
} else {
executeBulk(bulkRequest, startTime, listener, responses);
}
}