public void testVersioningWithBulk() {
createIndex("test");
ensureGreen();
BulkResponse bulkResponse =
client()
.prepareBulk()
.add(client().prepareIndex("test", "type", "1").setSource("field1", "value1_1"))
.execute()
.actionGet();
assertThat(bulkResponse.hasFailures(), equalTo(false));
assertThat(bulkResponse.getItems().length, equalTo(1));
IndexResponse indexResponse = bulkResponse.getItems()[0].getResponse();
assertThat(indexResponse.getVersion(), equalTo(1L));
}
@Override
public void executeESBulkRequest(BulkRequestBuilder esBulk)
throws ElasticsearchException, BulkUpdatePartialFailureException {
BulkResponse response = esBulk.execute().actionGet();
if (response.hasFailures()) {
boolean containsSuccess = false;
int numOfFailures = 0;
for (BulkItemResponse bir : response.getItems()) {
if (!bir.isFailed()) {
containsSuccess = true;
} else {
numOfFailures++;
}
}
if (containsSuccess) {
throw new BulkUpdatePartialFailureException(response.buildFailureMessage(), numOfFailures);
} else {
throw new ElasticsearchException(
"Failed to completely execute ES index bulk update for "
+ numOfFailures
+ " commands: "
+ response.buildFailureMessage());
}
}
}
@Override
public void onResponse(BulkResponse response) {
semaphore.release();
counter.addAndGet(response.getItems().length);
for (BulkItemResponse item : response.getItems()) {
if (item.isFailed()) {
LOGGER.error(
"index [{}], type [{}], id [{}], message [{}]",
item.getIndex(),
item.getType(),
item.getId(),
item.getFailureMessage());
}
}
}
@Test
public void testVersioningWithBulk() {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (IndexMissingException e) {
// its ok
}
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
BulkResponse bulkResponse =
client
.prepareBulk()
.add(client.prepareIndex("test", "type", "1").setSource("field1", "value1_1"))
.execute()
.actionGet();
assertThat(bulkResponse.hasFailures(), equalTo(false));
assertThat(bulkResponse.getItems().length, equalTo(1));
IndexResponse indexResponse = bulkResponse.getItems()[0].getResponse();
assertThat(indexResponse.version(), equalTo(1l));
}
/**
* This method processes failures by iterating through each bulk response item
*
* @param response, a BulkResponse
*/
private void processBulkResponseFailure(BulkResponse response) {
logger.warn("There was failures when executing bulk : " + response.buildFailureMessage());
if (!logger.isDebugEnabled()) return;
for (BulkItemResponse item : response.getItems()) {
if (item.isFailed()) {
logger.debug(
"Error {} occurred on index {}, type {}, id {} for {} operation ",
item.getFailureMessage(),
item.getIndex(),
item.getType(),
item.getId(),
item.getOpType());
}
}
}
@Override
public void bulkIndex(List<IndexQuery> queries) {
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (IndexQuery query : queries) {
bulkRequest.add(prepareIndex(query));
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
Map<String, String> failedDocuments = new HashMap<String, String>();
for (BulkItemResponse item : bulkResponse.getItems()) {
if (item.isFailed()) failedDocuments.put(item.getId(), item.getFailureMessage());
}
throw new ElasticsearchException(
"Bulk indexing has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages ["
+ failedDocuments
+ "]",
failedDocuments);
}
}
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
if (response.hasFailures()) {
throw new RuntimeException(response.buildFailureMessage());
}
indexedDocumentCount.addAndGet(response.getItems().length);
}
public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exception {
// Indexing into a should succeed, because the field mapping for field 'field' is defined in the
// test mapping.
client()
.admin()
.indices()
.preparePutTemplate("template1")
.setTemplate("a*")
.setOrder(0)
.addMapping("test", "field", "type=text")
.addAlias(new Alias("alias1").filter(termQuery("field", "value")))
.get();
// Indexing into b should succeed, because the field mapping for field 'field' is defined in the
// _default_ mapping and the test type exists.
client()
.admin()
.indices()
.preparePutTemplate("template2")
.setTemplate("b*")
.setOrder(0)
.addMapping("_default_", "field", "type=text")
.addMapping("test")
.addAlias(new Alias("alias2").filter(termQuery("field", "value")))
.get();
// Indexing into c should succeed, because the field mapping for field 'field' is defined in the
// _default_ mapping.
client()
.admin()
.indices()
.preparePutTemplate("template3")
.setTemplate("c*")
.setOrder(0)
.addMapping("_default_", "field", "type=text")
.addAlias(new Alias("alias3").filter(termQuery("field", "value")))
.get();
// Indexing into d index should fail, since there is field with name 'field' in the mapping
client()
.admin()
.indices()
.preparePutTemplate("template4")
.setTemplate("d*")
.setOrder(0)
.addAlias(new Alias("alias4").filter(termQuery("field", "value")))
.get();
client().prepareIndex("a1", "test", "test").setSource("{}").get();
BulkResponse response =
client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}")).get();
assertThat(response.hasFailures(), is(false));
assertThat(response.getItems()[0].isFailed(), equalTo(false));
assertThat(response.getItems()[0].getIndex(), equalTo("a2"));
assertThat(response.getItems()[0].getType(), equalTo("test"));
assertThat(response.getItems()[0].getId(), equalTo("test"));
assertThat(response.getItems()[0].getVersion(), equalTo(1L));
client().prepareIndex("b1", "test", "test").setSource("{}").get();
response =
client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}")).get();
assertThat(response.hasFailures(), is(false));
assertThat(response.getItems()[0].isFailed(), equalTo(false));
assertThat(response.getItems()[0].getIndex(), equalTo("b2"));
assertThat(response.getItems()[0].getType(), equalTo("test"));
assertThat(response.getItems()[0].getId(), equalTo("test"));
assertThat(response.getItems()[0].getVersion(), equalTo(1L));
client().prepareIndex("c1", "test", "test").setSource("{}").get();
response =
client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}")).get();
assertThat(response.hasFailures(), is(false));
assertThat(response.getItems()[0].isFailed(), equalTo(false));
assertThat(response.getItems()[0].getIndex(), equalTo("c2"));
assertThat(response.getItems()[0].getType(), equalTo("test"));
assertThat(response.getItems()[0].getId(), equalTo("test"));
assertThat(response.getItems()[0].getVersion(), equalTo(1L));
// Before 2.0 alias filters were parsed at alias creation time, in order
// for filters to work correctly ES required that fields mentioned in those
// filters exist in the mapping.
// From 2.0 and higher alias filters are parsed at request time and therefor
// fields mentioned in filters don't need to exist in the mapping.
// So the aliases defined in the index template for this index will not fail
// even though the fields in the alias fields don't exist yet and indexing into
// an index that doesn't exist yet will succeed
client().prepareIndex("d1", "test", "test").setSource("{}").get();
response =
client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}")).get();
assertThat(response.hasFailures(), is(false));
assertThat(response.getItems()[0].isFailed(), equalTo(false));
assertThat(response.getItems()[0].getId(), equalTo("test"));
assertThat(response.getItems()[0].getVersion(), equalTo(1L));
}
@Test
public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exception {
// Indexing into a should succeed, because the field mapping for field 'field' is defined in the
// test mapping.
client()
.admin()
.indices()
.preparePutTemplate("template1")
.setTemplate("a*")
.setOrder(0)
.addMapping("test", "field", "type=string")
.addAlias(new Alias("alias1").filter(termFilter("field", "value")))
.get();
// Indexing into b should succeed, because the field mapping for field 'field' is defined in the
// _default_ mapping and the test type exists.
client()
.admin()
.indices()
.preparePutTemplate("template2")
.setTemplate("b*")
.setOrder(0)
.addMapping("_default_", "field", "type=string")
.addMapping("test")
.addAlias(new Alias("alias2").filter(termFilter("field", "value")))
.get();
// Indexing into c should succeed, because the field mapping for field 'field' is defined in the
// _default_ mapping.
client()
.admin()
.indices()
.preparePutTemplate("template3")
.setTemplate("c*")
.setOrder(0)
.addMapping("_default_", "field", "type=string")
.addAlias(new Alias("alias3").filter(termFilter("field", "value")))
.get();
// Indexing into d index should fail, since there is field with name 'field' in the mapping
client()
.admin()
.indices()
.preparePutTemplate("template4")
.setTemplate("d*")
.setOrder(0)
.addAlias(new Alias("alias4").filter(termFilter("field", "value")))
.get();
client().prepareIndex("a1", "test", "test").setSource("{}").get();
BulkResponse response =
client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}")).get();
assertThat(response.hasFailures(), is(false));
assertThat(response.getItems()[0].isFailed(), equalTo(false));
assertThat(response.getItems()[0].getIndex(), equalTo("a2"));
assertThat(response.getItems()[0].getType(), equalTo("test"));
assertThat(response.getItems()[0].getId(), equalTo("test"));
assertThat(response.getItems()[0].getVersion(), equalTo(1l));
client().prepareIndex("b1", "test", "test").setSource("{}").get();
response =
client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}")).get();
assertThat(response.hasFailures(), is(false));
assertThat(response.getItems()[0].isFailed(), equalTo(false));
assertThat(response.getItems()[0].getIndex(), equalTo("b2"));
assertThat(response.getItems()[0].getType(), equalTo("test"));
assertThat(response.getItems()[0].getId(), equalTo("test"));
assertThat(response.getItems()[0].getVersion(), equalTo(1l));
client().prepareIndex("c1", "test", "test").setSource("{}").get();
response =
client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}")).get();
assertThat(response.hasFailures(), is(false));
assertThat(response.getItems()[0].isFailed(), equalTo(false));
assertThat(response.getItems()[0].getIndex(), equalTo("c2"));
assertThat(response.getItems()[0].getType(), equalTo("test"));
assertThat(response.getItems()[0].getId(), equalTo("test"));
assertThat(response.getItems()[0].getVersion(), equalTo(1l));
try {
client().prepareIndex("d1", "test", "test").setSource("{}").get();
fail();
} catch (Exception e) {
assertThat(
ExceptionsHelper.unwrapCause(e), instanceOf(ElasticsearchIllegalArgumentException.class));
assertThat(e.getMessage(), containsString("failed to parse filter for alias [alias4]"));
}
response =
client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}")).get();
assertThat(response.hasFailures(), is(true));
assertThat(response.getItems()[0].isFailed(), equalTo(true));
assertThat(
response.getItems()[0].getFailureMessage(),
containsString("failed to parse filter for alias [alias4]"));
}