@Override
public void executeESBulkRequest(BulkRequestBuilder esBulk)
throws ElasticsearchException, BulkUpdatePartialFailureException {
BulkResponse response = esBulk.execute().actionGet();
if (response.hasFailures()) {
boolean containsSuccess = false;
int numOfFailures = 0;
for (BulkItemResponse bir : response.getItems()) {
if (!bir.isFailed()) {
containsSuccess = true;
} else {
numOfFailures++;
}
}
if (containsSuccess) {
throw new BulkUpdatePartialFailureException(response.buildFailureMessage(), numOfFailures);
} else {
throw new ElasticsearchException(
"Failed to completely execute ES index bulk update for "
+ numOfFailures
+ " commands: "
+ response.buildFailureMessage());
}
}
}
@Test
public void testBulk_withIndexAndDeleteItems() throws Exception {
prepareIndex(1);
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
for (int i = 0; i < 10; i++) {
bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}"));
}
BulkResponse bulkResponse = bulkRequestBuilder.get();
bulkRequestBuilder = client().prepareBulk();
for (BulkItemResponse item : bulkResponse) {
assertThat(item.isFailed(), equalTo(false));
assertShardInfo(item.getResponse());
bulkRequestBuilder.add(client().prepareDelete("idx", "type", item.getId()));
}
bulkResponse = bulkRequestBuilder.get();
for (BulkItemResponse item : bulkResponse) {
assertThat(item.isFailed(), equalTo(false));
assertShardInfo(item.getResponse());
}
}
private void index(final QueuedTaskExecutor executor) {
final Esi4JBatchedEntityResolver entityResolver = executor.getEntityResolver();
if (entityResolver != null) {
entityResolver.resolveEntities(_tasks);
}
final BulkResponseWrapper response =
executor
.getTaskProcessor()
.getIndex()
.executeBulk(
new Esi4JOperation<ListenableActionFuture<BulkResponse>>() {
@Override
public ListenableActionFuture<BulkResponse> execute(
final Client client,
final String indexName,
final OperationContext helper) {
final BulkRequestBuilder bulk = client.prepareBulk();
for (final Esi4JEntityTask _task : _tasks) {
if (_task != null) {
_task.addToBulk(client, bulk, indexName, helper);
}
}
final ListenableActionFuture<BulkResponse> response = bulk.execute();
return response;
}
})
.actionGet();
int failed = 0;
for (final BulkItemResponse item : response.getBulkResponse()) {
if (item.isFailed()) {
failed++;
}
}
if (failed > 0) {
log.warn("failed to index " + failed + " items. index might be out of sync");
}
if (log.isDebugEnabled()) {
final int indexed = response.getBulkResponse().getItems().length - failed;
log.debug("finished bulk indexing " + indexed + " items");
}
}
protected void executeBulkRequest(BulkRequestBuilder bulkRequest) {
if (bulkRequest.numberOfActions() == 0) return;
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (!bulkResponse.hasFailures()) return;
for (BulkItemResponse response : bulkResponse) {
if (!response.isFailed()) continue;
LOG.warning(
String.format(
"Unable to save Entity %s in %s/%s, cause: %s",
response.getId(),
response.getIndex(),
response.getType(),
response.getFailureMessage()));
}
}
@Override
public void onResponse(BulkResponse response) {
semaphore.release();
counter.addAndGet(response.getItems().length);
for (BulkItemResponse item : response.getItems()) {
if (item.isFailed()) {
LOGGER.error(
"index [{}], type [{}], id [{}], message [{}]",
item.getIndex(),
item.getType(),
item.getId(),
item.getFailureMessage());
}
}
}
/**
* This method processes failures by iterating through each bulk response item
*
* @param response, a BulkResponse
*/
private void processBulkResponseFailure(BulkResponse response) {
logger.warn("There was failures when executing bulk : " + response.buildFailureMessage());
if (!logger.isDebugEnabled()) return;
for (BulkItemResponse item : response.getItems()) {
if (item.isFailed()) {
logger.debug(
"Error {} occurred on index {}, type {}, id {} for {} operation ",
item.getFailureMessage(),
item.getIndex(),
item.getType(),
item.getId(),
item.getOpType());
}
}
}
@Test
public void testBulk_withUpdateItems() throws Exception {
prepareIndex(1);
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
for (int i = 0; i < 10; i++) {
bulkRequestBuilder.add(
client()
.prepareUpdate("idx", "type", Integer.toString(i))
.setDoc("{}")
.setDocAsUpsert(true));
}
BulkResponse bulkResponse = bulkRequestBuilder.get();
for (BulkItemResponse item : bulkResponse) {
assertThat(item.isFailed(), equalTo(false));
assertShardInfo(item.getResponse());
}
}
@Override
public void bulkIndex(List<IndexQuery> queries) {
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (IndexQuery query : queries) {
bulkRequest.add(prepareIndex(query));
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
Map<String, String> failedDocuments = new HashMap<String, String>();
for (BulkItemResponse item : bulkResponse.getItems()) {
if (item.isFailed()) failedDocuments.put(item.getId(), item.getFailureMessage());
}
throw new ElasticsearchException(
"Bulk indexing has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages ["
+ failedDocuments
+ "]",
failedDocuments);
}
}
/**
* Updates the specified objects
*
* @return the id's of the failed objects (e.g. due to versioning)
*/
public Collection<Integer> bulkUpdate(
Collection<T> objects, String indexName, boolean refresh, boolean enableVersioning) {
// now using bulk API instead of feeding each doc separate with feedDoc
BulkRequestBuilder brb = client.prepareBulk();
// this works differently then the direct call to refresh!? maybe refresh is not async?
// brb.setRefresh(refresh);
for (T o : objects) {
if (o.getId() == null) {
logger.warn("Skipped object without id when bulkUpdate:" + o);
continue;
}
try {
XContentBuilder source = createDoc(o);
IndexRequest indexReq =
Requests.indexRequest(indexName).type(getIndexType()).id(o.getId()).source(source);
if (enableVersioning) indexReq.version(o.getVersion());
brb.add(indexReq);
} catch (IOException ex) {
logger.warn("Cannot add object:" + o + " to bulkIndexing action." + ex.getMessage());
}
}
if (brb.numberOfActions() > 0) {
BulkResponse rsp = brb.execute().actionGet();
if (rsp.hasFailures()) {
List<Integer> list = new ArrayList<Integer>(rsp.items().length);
for (BulkItemResponse br : rsp.items()) {
if (br.isFailed()) {
// logger.info("Error:" + br.failureMessage());
list.add(br.itemId());
}
}
return list;
}
if (refresh) refresh(indexName);
}
return Collections.emptyList();
}