public void testForce() throws Exception {
createIndex("test");
ensureGreen(
"test"); // we are testing force here which doesn't work if we are recovering at the same
// time - zzzzz...
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(12)
.setVersionType(VersionType.FORCE)
.get();
assertThat(indexResponse.getVersion(), equalTo(12L));
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_2")
.setVersion(12)
.setVersionType(VersionType.FORCE)
.get();
assertThat(indexResponse.getVersion(), equalTo(12L));
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_2")
.setVersion(14)
.setVersionType(VersionType.FORCE)
.get();
assertThat(indexResponse.getVersion(), equalTo(14L));
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(13)
.setVersionType(VersionType.FORCE)
.get();
assertThat(indexResponse.getVersion(), equalTo(13L));
client().admin().indices().prepareRefresh().execute().actionGet();
if (randomBoolean()) {
refresh();
}
for (int i = 0; i < 10; i++) {
assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(13L));
}
// deleting with a lower version works.
long v = randomIntBetween(12, 14);
DeleteResponse deleteResponse =
client()
.prepareDelete("test", "type", "1")
.setVersion(v)
.setVersionType(VersionType.FORCE)
.get();
assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
assertThat(deleteResponse.getVersion(), equalTo(v));
}
@Override
public String importQuestionnareOds(String filename, String questionnaireName) throws Exception {
SpreadsheetDocument document = SpreadsheetDocument.loadDocument(new File(filename));
Table sheet = document.getSheetByIndex(0);
List<String> questions = new ArrayList<>();
for (int i = 1; i < sheet.getRowList().size(); i++) {
Row row = sheet.getRowList().get(i);
String question = getCellStringValue(row, 0);
if (StringUtils.isBlank(question)) {
break;
}
String levelString = getCellStringValue(row, 1);
long level = Long.valueOf(StringUtils.replace(levelString, "D", ""));
String tagsString = getCellStringValue(row, 2);
List<String> tags = Collections.emptyList();
if (StringUtils.isNotBlank(tagsString)) {
tags = Lists.newArrayList(StringUtils.split(tagsString, ", "));
}
String tip = StringUtils.defaultIfBlank(getCellStringValue(row, 3), null);
XContentBuilder builder =
jsonBuilder()
.startObject()
.field("title", question)
.field("level", level)
.field("tags", tags)
.field("tip", tip)
.endObject();
IndexResponse indexResponse =
client
.prepareIndex(domainResolver.resolveQuestionIndex(), Types.question)
.setSource(builder)
.execute()
.actionGet();
questions.add(indexResponse.getId());
}
XContentBuilder questionnaireBuilder =
jsonBuilder()
.startObject()
.field("name", questionnaireName)
.field("questions", questions)
.endObject();
IndexResponse indexResponse =
client
.prepareIndex(domainResolver.resolveQuestionIndex(), Types.questionnaire)
.setSource(questionnaireBuilder)
.execute()
.actionGet();
return indexResponse.getId();
}
@Override
public void notify(DelegateExecution execution) throws Exception {
HistoryService historyService = execution.getProcessEngineServices().getHistoryService();
HistoricDecisionInstance historicDecisionInstance =
historyService
.createHistoricDecisionInstanceQuery()
.includeInputs()
.includeOutputs()
.decisionDefinitionKey((String) execution.getVariableLocal("tableName"))
.processInstanceId(execution.getProcessInstanceId())
.singleResult();
// Fill a new object with stuff...
FraudScoreTableObject fraudData = new FraudScoreTableObject();
fraudData.setFraudInstanceID(historicDecisionInstance.getId());
List<HistoricDecisionInputInstance> inputs = historicDecisionInstance.getInputs();
for (HistoricDecisionInputInstance historicDecisionInputInstance : inputs) {
String inputName = historicDecisionInputInstance.getTypeName();
if (inputName.equals("paymentRejected")) {
fraudData.setPaymentRejected((Boolean) historicDecisionInputInstance.getValue());
} else if (inputName.equals("numberOfPayouts")) {
fraudData.setNumberOfPayouts((Integer) historicDecisionInputInstance.getValue());
} else if (inputName.equals("historyOfFraud")) {
fraudData.setHistoryOfFraud((Boolean) historicDecisionInputInstance.getValue());
} else if (inputName.equals("claimAmount")) {
fraudData.setCalimAmount((Long) historicDecisionInputInstance.getValue());
}
}
List<HistoricDecisionOutputInstance> outputs = historicDecisionInstance.getOutputs();
for (HistoricDecisionOutputInstance historicDecisionOutputInstance : outputs) {
String inputName = historicDecisionOutputInstance.getTypeName();
if (inputName.equals("frausScore")) {
fraudData.setFraudScore((Integer) historicDecisionOutputInstance.getValue());
}
}
ObjectMapper mapper = new ObjectMapper();
String serializedHistoricDecisionInstance =
mapper.writerWithDefaultPrettyPrinter().writeValueAsString(fraudData);
Client client =
TransportClient.builder()
.build()
.addTransportAddress(
new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300));
IndexResponse response =
client
.prepareIndex("camunda", "fraudData", historicDecisionInstance.getId())
.setSource(serializedHistoricDecisionInstance)
.get();
LOGGER.info(response.getId());
}
public void index(final Entity entity, final Tenant tenant) throws SearchEngineException {
try {
Map<String, Object> source = entityIndexDocumentPurveyor.purveyDocument(entity, tenant);
if (source.keySet().size() > 0) {
String entityName =
StringUtils.substringAfterLast(entity.getClass().getName(), ".").toLowerCase();
this.logger.debug("Indexing {} with id {}...", entityName, entity.getId());
IndexRequestBuilder builder =
this.client
.prepareIndex("entities", entityName, entity.getId().toString())
.setSource(source);
if (tenant != null && !Tenant.class.isAssignableFrom(entity.getClass())) {
builder.setParent(tenant.getId().toString());
}
IndexResponse response = builder.execute().actionGet();
this.logger.debug("" + response.getType());
}
} catch (Exception e) {
throw new SearchEngineException("Failed to index entity", e);
}
}
@Test
public void testIndexAndDelete() throws Exception {
prepareIndex(1);
IndexResponse indexResponse = client().prepareIndex("idx", "type").setSource("{}").get();
assertShardInfo(indexResponse);
DeleteResponse deleteResponse =
client().prepareDelete("idx", "type", indexResponse.getId()).get();
assertShardInfo(deleteResponse);
}
public int indexDocs(final int numOfDoc) throws Exception {
for (int doc = 0; doc < numOfDoc; doc++) {
final IndexRequest indexRequest =
new IndexRequest(index.getName(), "type", Integer.toString(docId.incrementAndGet()))
.source("{}");
final IndexResponse response = index(indexRequest);
assertEquals(DocWriteResponse.Result.CREATED, response.getResult());
}
return numOfDoc;
}
public void testVersioningWithBulk() {
createIndex("test");
ensureGreen();
BulkResponse bulkResponse =
client()
.prepareBulk()
.add(client().prepareIndex("test", "type", "1").setSource("field1", "value1_1"))
.execute()
.actionGet();
assertThat(bulkResponse.hasFailures(), equalTo(false));
assertThat(bulkResponse.getItems().length, equalTo(1));
IndexResponse indexResponse = bulkResponse.getItems()[0].getResponse();
assertThat(indexResponse.getVersion(), equalTo(1L));
}
protected void registerRiver(final String typename, final String file)
throws ElasticsearchException, IOException {
final IndexResponse res =
esSetup
.client()
.prepareIndex()
.setIndex("_river")
.setType(typename)
.setId("_meta")
.setSource(loadFile(file))
.execute()
.actionGet();
if (!res.isCreated()) {
throw new IOException("Unable to register river");
}
}
public static String save(
ElasticSearchContextListener es, String userId, String memeId, String value) {
try {
String pendingItemId = getPendingItemId(es, userId, memeId);
Map<String, Object> data = getRatingsObjectMap(userId, memeId, value);
if (pendingItemId != null) {
UpdateResponse response = es.updateRequest(INDEX_NAME, pendingItemId, data).actionGet();
return response.getId();
} else {
IndexResponse response = es.indexRequest(INDEX_NAME, data).actionGet();
return response.getId();
}
} catch (Exception e) {
return null;
}
}
public void testInternalVersioningInitialDelete() throws Exception {
createIndex("test");
ensureGreen();
assertThrows(
client().prepareDelete("test", "type", "1").setVersion(17).execute(),
VersionConflictEngineException.class);
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setCreate(true)
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(1L));
}
/**
* Saves an entity into elastic search.
*
* @param object The entity to save.
*/
public void save(T object) {
try {
IndexRequestBuilder indexRequestBuilder =
object.getId() == null
? client.prepareIndex(index, entity)
: client.prepareIndex(index, entity, object.getId());
IndexResponse response =
indexRequestBuilder.setSource(mapper.writeValueAsString(object)).execute().actionGet();
// Force refresh.
object.setId(response.getId());
client.admin().indices().prepareRefresh(index).execute().actionGet();
if (logger.isDebugEnabled()) {
BytesStreamOutput out = new BytesStreamOutput(1024 * 1024);
response.writeTo(out);
logger.debug("Obtained response: {}", new String(out.bytes().array()));
}
} catch (ElasticSearchException | IOException e) {
logger.error("Error while saving entity", e);
}
}
@Override
public void process(final Exchange exchange) throws Exception {
final String indexName = exchange.getIn().getHeader(INDEX_NAME, String.class);
final String indexSchema = exchange.getIn().getHeader(INDEX_SCHEMA, String.class);
final Indexer indexer = exchange.getIn().getHeader(INDEXER, Indexer.class);
final Map data = exchange.getIn().getBody(Map.class);
data.put(
Constants.CORRELATION_ID,
exchange.getIn().getHeader(Constants.CORRELATION_ID, String.class));
final String serValue = mapper.writeValueAsString(data);
LOG.debug(serValue);
final IndexResponse response =
indexer
.es()
.prepareIndex(indexName, indexSchema)
.setSource(serValue)
.execute()
.get(1, TimeUnit.MINUTES);
LOG.debug(response.toString());
}
@Test
public void testVersioningWithBulk() {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (IndexMissingException e) {
// its ok
}
client.admin().indices().prepareCreate("test").execute().actionGet();
client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
BulkResponse bulkResponse =
client
.prepareBulk()
.add(client.prepareIndex("test", "type", "1").setSource("field1", "value1_1"))
.execute()
.actionGet();
assertThat(bulkResponse.hasFailures(), equalTo(false));
assertThat(bulkResponse.getItems().length, equalTo(1));
IndexResponse indexResponse = bulkResponse.getItems()[0].getResponse();
assertThat(indexResponse.version(), equalTo(1l));
}
public void testIndexGetAndDelete() throws ExecutionException, InterruptedException {
createIndexWithAlias();
ensureYellow("test");
int numDocs = iterations(10, 50);
for (int i = 0; i < numDocs; i++) {
IndexResponse indexResponse =
client()
.prepareIndex(indexOrAlias(), "type", Integer.toString(i))
.setSource("field", "value-" + i)
.get();
assertThat(indexResponse.isCreated(), equalTo(true));
assertThat(indexResponse.getIndex(), equalTo("test"));
assertThat(indexResponse.getType(), equalTo("type"));
assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
}
refresh();
String docId = Integer.toString(randomIntBetween(0, numDocs - 1));
GetResponse getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
assertThat(getResponse.isExists(), equalTo(true));
assertThat(getResponse.getIndex(), equalTo("test"));
assertThat(getResponse.getType(), equalTo("type"));
assertThat(getResponse.getId(), equalTo(docId));
DeleteResponse deleteResponse = client().prepareDelete(indexOrAlias(), "type", docId).get();
assertThat(deleteResponse.isFound(), equalTo(true));
assertThat(deleteResponse.getIndex(), equalTo("test"));
assertThat(deleteResponse.getType(), equalTo("type"));
assertThat(deleteResponse.getId(), equalTo(docId));
getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
assertThat(getResponse.isExists(), equalTo(false));
refresh();
SearchResponse searchResponse = client().prepareSearch(indexOrAlias()).get();
assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1));
}
public void testExternalVersioningInitialDelete() throws Exception {
createIndex("test");
ensureGreen();
// Note - external version doesn't throw version conflicts on deletes of non existent records.
// This is different from internal versioning
DeleteResponse deleteResponse =
client()
.prepareDelete("test", "type", "1")
.setVersion(17)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertEquals(DocWriteResponse.Operation.NOOP, deleteResponse.getOperation());
// this should conflict with the delete command transaction which told us that the object was
// deleted at version 17.
assertThrows(
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(13)
.setVersionType(VersionType.EXTERNAL)
.execute(),
VersionConflictEngineException.class);
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(18)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(18L));
}
public static ObjectNode toJSON(ObjectMapper objectMapper, RecomputeResult success)
throws InterruptedException, ExecutionException {
final ObjectNode n = objectMapper.createObjectNode();
// write targetID using index reponse
final IndexResponse ir = success.indexResponse();
final ObjectNode targetID = objectMapper.createObjectNode();
targetID.put("index", ir.index());
targetID.put("type", ir.type());
targetID.put("id", ir.id());
targetID.put("version", ir.version());
n.put("targetID", targetID);
// write dictionary of properties and their values
final ObjectNode properties = objectMapper.createObjectNode();
for (Map.Entry<String, JsonNode> entry : success.properties().entrySet())
properties.put(entry.getKey(), entry.getValue());
n.put("properties", properties);
// write dictionary of properties and their values
n.put("sourceDocumentAfterTransform", success.sourceDocument());
return n;
}
public static int Record(long sync_id, String idx) {
LogLine temp = rec_map.get(sync_id);
if (null == temp) {
idx_map.put(sync_id, idx);
} else {
temp.dstidx = idx;
// tell score system about temp;
rec_map.remove(sync_id);
// tell elasticsearch;
IndexResponse response =
client
.prepareIndex(DATABASEINDEX, DATABASETYPE)
.setSource(gson.toJson(temp))
.execute()
.actionGet();
if (!response.isCreated()) {
System.out.println(PROMPT + "Elasticsearch recording failed!");
return ERROR;
}
// test
WriteToDisk(temp);
}
return SUCCESS;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("id=");
sb.append(id);
sb.append(" version=");
sb.append(version);
sb.append(" delete?=");
sb.append(delete);
sb.append(" threadID=");
sb.append(threadID);
sb.append(" indexStartTime=");
sb.append(indexStartTime);
sb.append(" indexFinishTime=");
sb.append(indexFinishTime);
sb.append(" versionConflict=");
sb.append(versionConflict);
sb.append(" alreadyExists?=");
sb.append(alreadyExists);
if (response != null) {
if (response instanceof DeleteResponse) {
DeleteResponse deleteResponse = (DeleteResponse) response;
sb.append(" response:");
sb.append(" index=");
sb.append(deleteResponse.getIndex());
sb.append(" id=");
sb.append(deleteResponse.getId());
sb.append(" type=");
sb.append(deleteResponse.getType());
sb.append(" version=");
sb.append(deleteResponse.getVersion());
sb.append(" found=");
sb.append(deleteResponse.getOperation() == DocWriteResponse.Operation.DELETE);
} else if (response instanceof IndexResponse) {
IndexResponse indexResponse = (IndexResponse) response;
sb.append(" index=");
sb.append(indexResponse.getIndex());
sb.append(" id=");
sb.append(indexResponse.getId());
sb.append(" type=");
sb.append(indexResponse.getType());
sb.append(" version=");
sb.append(indexResponse.getVersion());
sb.append(" created=");
sb.append(indexResponse.getOperation() == DocWriteResponse.Operation.CREATE);
} else {
sb.append(" response: " + response);
}
} else {
sb.append(" response: null");
}
return sb.toString();
}
@Test
public void testIndex() {
IndexResponse response = null;
try {
response =
client
.prepareIndex("asdf2014", "asdf", "1")
.setSource(
jsonBuilder().startObject().field("age", 22).field("sex", "male").endObject())
.execute()
.actionGet();
} catch (ElasticsearchException | IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
System.out.println("Index: " + response.getIndex());
System.out.println("Type: " + response.getType());
System.out.println("Id: " + response.getId());
System.err.println("Version: " + response.getVersion());
System.out.println("IsCreated: " + response.isCreated());
}
@Test
public void testNetworkPartitionDuringReplicaIndexOp() throws Exception {
final String INDEX = "testidx";
List<String> nodes = internalCluster().startNodesAsync(2, nodeSettings).get();
// Create index test with 1 shard, 1 replica and ensure it is green
createIndex(INDEX);
ensureGreen(INDEX);
// Disable allocation so the replica cannot be reallocated when it fails
Settings s =
ImmutableSettings.builder().put("cluster.routing.allocation.enable", "none").build();
client().admin().cluster().prepareUpdateSettings().setTransientSettings(s).get();
// Determine which node holds the primary shard
ClusterState state = getNodeClusterState(nodes.get(0));
IndexShardRoutingTable shard = state.getRoutingTable().index(INDEX).shard(0);
String primaryNode;
String replicaNode;
if (shard.getShards().get(0).primary()) {
primaryNode = nodes.get(0);
replicaNode = nodes.get(1);
} else {
primaryNode = nodes.get(1);
replicaNode = nodes.get(0);
}
logger.info("--> primary shard is on {}", primaryNode);
// Index a document to make sure everything works well
IndexResponse resp =
internalCluster()
.client(primaryNode)
.prepareIndex(INDEX, "doc")
.setSource("foo", "bar")
.get();
assertThat(
"document exists on primary node",
internalCluster()
.client(primaryNode)
.prepareGet(INDEX, "doc", resp.getId())
.setPreference("_only_local")
.get()
.isExists(),
equalTo(true));
assertThat(
"document exists on replica node",
internalCluster()
.client(replicaNode)
.prepareGet(INDEX, "doc", resp.getId())
.setPreference("_only_local")
.get()
.isExists(),
equalTo(true));
// Disrupt the network so indexing requests fail to replicate
logger.info("--> preventing index/replica operations");
TransportService mockTransportService =
internalCluster().getInstance(TransportService.class, primaryNode);
((MockTransportService) mockTransportService)
.addFailToSendNoConnectRule(
internalCluster().getInstance(Discovery.class, replicaNode).localNode(),
ImmutableSet.of(IndexAction.NAME + "[r]"));
mockTransportService = internalCluster().getInstance(TransportService.class, replicaNode);
((MockTransportService) mockTransportService)
.addFailToSendNoConnectRule(
internalCluster().getInstance(Discovery.class, primaryNode).localNode(),
ImmutableSet.of(IndexAction.NAME + "[r]"));
logger.info("--> indexing into primary");
// the replica shard should now be marked as failed because the replication operation will fail
resp =
internalCluster()
.client(primaryNode)
.prepareIndex(INDEX, "doc")
.setSource("foo", "baz")
.get();
// wait until the cluster reaches an exact yellow state, meaning replica has failed
assertBusy(
new Runnable() {
@Override
public void run() {
assertThat(
client().admin().cluster().prepareHealth().get().getStatus(),
equalTo(ClusterHealthStatus.YELLOW));
}
});
assertThat(
"document should still be indexed and available",
client().prepareGet(INDEX, "doc", resp.getId()).get().isExists(),
equalTo(true));
state = getNodeClusterState(randomFrom(nodes.toArray(Strings.EMPTY_ARRAY)));
RoutingNodes rn = state.routingNodes();
logger.info(
"--> counts: total: {}, unassigned: {}, initializing: {}, relocating: {}, started: {}",
rn.shards(
new Predicate<MutableShardRouting>() {
@Override
public boolean apply(
org.elasticsearch.cluster.routing.MutableShardRouting input) {
return true;
}
})
.size(),
rn.shardsWithState(UNASSIGNED).size(),
rn.shardsWithState(INITIALIZING).size(),
rn.shardsWithState(RELOCATING).size(),
rn.shardsWithState(STARTED).size());
logger.info(
"--> unassigned: {}, initializing: {}, relocating: {}, started: {}",
rn.shardsWithState(UNASSIGNED),
rn.shardsWithState(INITIALIZING),
rn.shardsWithState(RELOCATING),
rn.shardsWithState(STARTED));
assertThat(
"only a single shard is now active (replica should be failed and not reallocated)",
rn.shardsWithState(STARTED).size(),
equalTo(1));
}
@Test
public void testSimpleTTL() throws Exception {
assertAcked(
prepareCreate("test")
.addMapping(
"type1",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.endObject()
.endObject()
.endObject())
.addMapping(
"type2",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type2")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.field("default", "1d")
.endObject()
.endObject()
.endObject()));
ensureYellow("test");
final NumShards test = getNumShards("test");
long providedTTLValue = 3000;
logger.info("--> checking ttl");
// Index one doc without routing, one doc with routing, one doc with not TTL and no default and
// one doc with default TTL
long now = System.currentTimeMillis();
IndexResponse indexResponse =
client()
.prepareIndex("test", "type1", "1")
.setSource("field1", "value1")
.setTimestamp(String.valueOf(now))
.setTTL(providedTTLValue)
.setRefresh(true)
.get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client()
.prepareIndex("test", "type1", "with_routing")
.setSource("field1", "value1")
.setTimestamp(String.valueOf(now))
.setTTL(providedTTLValue)
.setRouting("routing")
.setRefresh(true)
.get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client().prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client().prepareIndex("test", "type2", "default_ttl").setSource("field1", "value1").get();
assertThat(indexResponse.isCreated(), is(true));
// realtime get check
long currentTime = System.currentTimeMillis();
GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
long ttl0;
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// verify the ttl is still decreasing when going to the replica
currentTime = System.currentTimeMillis();
getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// non realtime get (stored)
currentTime = System.currentTimeMillis();
getResponse =
client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// non realtime get going the replica
currentTime = System.currentTimeMillis();
getResponse =
client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// no TTL provided so no TTL fetched
getResponse =
client()
.prepareGet("test", "type1", "no_ttl")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.getField("_ttl"), nullValue());
// no TTL provided make sure it has default TTL
getResponse =
client()
.prepareGet("test", "type2", "default_ttl")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, greaterThan(0L));
IndicesStatsResponse response =
client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
assertThat(
response.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount(),
equalTo(0L));
// make sure the purger has done its job for all indexed docs that are expired
long shouldBeExpiredDate = now + providedTTLValue + PURGE_INTERVAL + 2000;
currentTime = System.currentTimeMillis();
if (shouldBeExpiredDate - currentTime > 0) {
Thread.sleep(shouldBeExpiredDate - currentTime);
}
// We can't assume that after waiting for ttl + purgeInterval (waitTime) that the document have
// actually been deleted.
// The ttl purging happens in the background in a different thread, and might not have been
// completed after waiting for waitTime.
// But we can use index statistics' delete count to be sure that deletes have been executed,
// that must be incremented before
// ttl purging has finished.
logger.info("--> checking purger");
assertThat(
awaitBusy(
new Predicate<Object>() {
@Override
public boolean apply(Object input) {
if (rarely()) {
client().admin().indices().prepareFlush("test").get();
} else if (rarely()) {
client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get();
}
IndicesStatsResponse response =
client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
// TTL deletes two docs, but it is indexed in the primary shard and replica shard.
return response
.getIndices()
.get("test")
.getTotal()
.getIndexing()
.getTotal()
.getDeleteCount()
== 2L * test.dataCopies;
}
},
5,
TimeUnit.SECONDS),
equalTo(true));
// realtime get check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// replica realtime get check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// Need to run a refresh, in order for the non realtime get to work.
client().admin().indices().prepareRefresh("test").execute().actionGet();
// non realtime get (stored) check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// non realtime get going the replica check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
}
@Override
public void setShardInfo(ReplicationResponse.ShardInfo shardInfo) {
finalResponse.setShardInfo(shardInfo);
}
@Test
public void fullTest() throws ExecutionException, InterruptedException, DegraphmalizerException {
final String target = "test-target";
final String idx = "test-index";
final String tp = "test-type";
final String id = "1234";
// create target index if it doesn't exist
if (!ln.es.admin().indices().prepareExists(target).execute().actionGet().exists()) {
final CreateIndexResponse cir =
ln.es.admin().indices().prepareCreate(target).execute().actionGet();
if (!cir.acknowledged()) throw new RuntimeException("failed to create index " + target);
}
final IndexResponse ir =
ln.es.prepareIndex(idx, tp, id).setSource("{\"children\":[1,2,3]}").execute().actionGet();
ln.log.info(
"Indexed /{}/{}/{} as version {} into ES", new Object[] {idx, tp, id, ir.version()});
final IndexResponse ir1 =
ln.es
.prepareIndex(idx, tp, "1")
.setSource("{\"cheese\":\"gorgonzola\"}")
.execute()
.actionGet();
ln.log.info("Indexed /{}/{}/1 as version {} into ES", new Object[] {idx, tp, ir1.version()});
final IndexResponse ir2 =
ln.es
.prepareIndex(idx, tp, "2")
.setSource("{\"cheese\":\"mozarella\"}")
.execute()
.actionGet();
ln.log.info("Indexed /{}/{}/2 as version {} into ES", new Object[] {idx, tp, ir2.version()});
// degraphmalize "1" and wait for and print result
final List<Future<DegraphmalizeResult>> actions = new ArrayList<Future<DegraphmalizeResult>>();
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.UPDATE,
DegraphmalizeRequestScope.DOCUMENT,
new ID(idx, tp, id, ir.version()),
ln.callback));
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.UPDATE,
DegraphmalizeRequestScope.DOCUMENT,
new ID(idx, tp, "1", ir1.version()),
ln.callback));
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.UPDATE,
DegraphmalizeRequestScope.DOCUMENT,
new ID(idx, tp, "2", ir2.version()),
ln.callback));
for (final Future<DegraphmalizeResult> a : actions) {
DegraphmalizeResult result = a.get();
ln.log.info("Degraphmalize complete for : " + result.root());
}
ObjectMapper mapper = new ObjectMapper();
for (final Future<DegraphmalizeResult> a : actions) {
final DegraphmalizeResult degraphmalizeResult = a.get();
// Get first node in results
final ObjectNode result = toJSON(mapper, degraphmalizeResult.results().get(0).get());
assertThat(result.get("properties").has("nodes-in")).isTrue();
assertThat(result.get("properties").has("nodes-out")).isTrue();
if (degraphmalizeResult.root().id().equals("1234")) {
assertThat(numberOfChildren(result, "nodes-out")).isZero();
assertThat(numberOfChildren(result, "nodes-in")).isEqualTo(3);
}
if (degraphmalizeResult.root().id().equals("1")) {
assertThat(numberOfChildren(result, "nodes-out")).isEqualTo(1);
assertThat(numberOfChildren(result, "nodes-in")).isZero();
}
}
GraphUtilities.dumpGraph(new ObjectMapper(), ln.G);
// Cleanup index
if (!ln.es.admin().indices().delete(new DeleteIndexRequest(idx)).actionGet().acknowledged()) {
throw new RuntimeException("failed to delete index " + target);
}
}
public void testExternalGTE() throws Exception {
createIndex("test");
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(12)
.setVersionType(VersionType.EXTERNAL_GTE)
.get();
assertThat(indexResponse.getVersion(), equalTo(12L));
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_2")
.setVersion(12)
.setVersionType(VersionType.EXTERNAL_GTE)
.get();
assertThat(indexResponse.getVersion(), equalTo(12L));
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_2")
.setVersion(14)
.setVersionType(VersionType.EXTERNAL_GTE)
.get();
assertThat(indexResponse.getVersion(), equalTo(14L));
assertThrows(
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(13)
.setVersionType(VersionType.EXTERNAL_GTE),
VersionConflictEngineException.class);
client().admin().indices().prepareRefresh().execute().actionGet();
if (randomBoolean()) {
refresh();
}
for (int i = 0; i < 10; i++) {
assertThat(client().prepareGet("test", "type", "1").get().getVersion(), equalTo(14L));
}
// deleting with a lower version fails.
assertThrows(
client()
.prepareDelete("test", "type", "1")
.setVersion(2)
.setVersionType(VersionType.EXTERNAL_GTE),
VersionConflictEngineException.class);
// Delete with a higher or equal version deletes all versions up to the given one.
long v = randomIntBetween(14, 17);
DeleteResponse deleteResponse =
client()
.prepareDelete("test", "type", "1")
.setVersion(v)
.setVersionType(VersionType.EXTERNAL_GTE)
.execute()
.actionGet();
assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
assertThat(deleteResponse.getVersion(), equalTo(v));
// Deleting with a lower version keeps on failing after a delete.
assertThrows(
client()
.prepareDelete("test", "type", "1")
.setVersion(2)
.setVersionType(VersionType.EXTERNAL_GTE)
.execute(),
VersionConflictEngineException.class);
// But delete with a higher version is OK.
deleteResponse =
client()
.prepareDelete("test", "type", "1")
.setVersion(18)
.setVersionType(VersionType.EXTERNAL_GTE)
.execute()
.actionGet();
assertEquals(DocWriteResponse.Operation.NOOP, deleteResponse.getOperation());
assertThat(deleteResponse.getVersion(), equalTo(18L));
}
public void testSimpleVersioningWithFlush() throws Exception {
createIndex("test");
ensureGreen();
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(1L));
client().admin().indices().prepareFlush().execute().actionGet();
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_2")
.setVersion(1)
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(2L));
client().admin().indices().prepareFlush().execute().actionGet();
assertThrows(
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(1)
.execute(),
VersionConflictEngineException.class);
assertThrows(
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(1)
.execute(),
VersionConflictEngineException.class);
assertThrows(
client()
.prepareIndex("test", "type", "1")
.setCreate(true)
.setSource("field1", "value1_1")
.execute(),
VersionConflictEngineException.class);
assertThrows(
client().prepareDelete("test", "type", "1").setVersion(1).execute(),
VersionConflictEngineException.class);
assertThrows(
client().prepareDelete("test", "type", "1").setVersion(1).execute(),
VersionConflictEngineException.class);
client().admin().indices().prepareRefresh().execute().actionGet();
for (int i = 0; i < 10; i++) {
assertThat(
client().prepareGet("test", "type", "1").execute().actionGet().getVersion(), equalTo(2L));
}
for (int i = 0; i < 10; i++) {
SearchResponse searchResponse =
client().prepareSearch().setQuery(matchAllQuery()).setVersion(true).execute().actionGet();
assertThat(searchResponse.getHits().getAt(0).version(), equalTo(2L));
}
}
public void testInternalVersioning() throws Exception {
createIndex("test");
ensureGreen();
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(1L));
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_2")
.setVersion(1)
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(2L));
assertThrows(
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(1)
.execute(),
VersionConflictEngineException.class);
assertThrows(
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(1)
.execute(),
VersionConflictEngineException.class);
assertThrows(
client()
.prepareIndex("test", "type", "1")
.setCreate(true)
.setSource("field1", "value1_1")
.execute(),
VersionConflictEngineException.class);
assertThrows(
client().prepareDelete("test", "type", "1").setVersion(1).execute(),
VersionConflictEngineException.class);
assertThrows(
client().prepareDelete("test", "type", "1").setVersion(1).execute(),
VersionConflictEngineException.class);
client().admin().indices().prepareRefresh().execute().actionGet();
for (int i = 0; i < 10; i++) {
assertThat(
client().prepareGet("test", "type", "1").execute().actionGet().getVersion(), equalTo(2L));
}
// search with versioning
for (int i = 0; i < 10; i++) {
SearchResponse searchResponse =
client().prepareSearch().setQuery(matchAllQuery()).setVersion(true).execute().actionGet();
assertThat(searchResponse.getHits().getAt(0).version(), equalTo(2L));
}
// search without versioning
for (int i = 0; i < 10; i++) {
SearchResponse searchResponse =
client().prepareSearch().setQuery(matchAllQuery()).execute().actionGet();
assertThat(searchResponse.getHits().getAt(0).version(), equalTo(Versions.NOT_FOUND));
}
DeleteResponse deleteResponse =
client().prepareDelete("test", "type", "1").setVersion(2).execute().actionGet();
assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
assertThat(deleteResponse.getVersion(), equalTo(3L));
assertThrows(
client().prepareDelete("test", "type", "1").setVersion(2).execute(),
VersionConflictEngineException.class);
// This is intricate - the object was deleted but a delete transaction was with the right
// version. We add another one
// and thus the transaction is increased.
deleteResponse =
client().prepareDelete("test", "type", "1").setVersion(3).execute().actionGet();
assertEquals(DocWriteResponse.Operation.NOOP, deleteResponse.getOperation());
assertThat(deleteResponse.getVersion(), equalTo(4L));
}
public void testExternalVersioning() throws Exception {
createIndex("test");
ensureGreen();
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(12)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(12L));
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(14)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(14L));
assertThrows(
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(13)
.setVersionType(VersionType.EXTERNAL)
.execute(),
VersionConflictEngineException.class);
if (randomBoolean()) {
refresh();
}
for (int i = 0; i < 10; i++) {
assertThat(
client().prepareGet("test", "type", "1").execute().actionGet().getVersion(),
equalTo(14L));
}
// deleting with a lower version fails.
assertThrows(
client()
.prepareDelete("test", "type", "1")
.setVersion(2)
.setVersionType(VersionType.EXTERNAL)
.execute(),
VersionConflictEngineException.class);
// Delete with a higher version deletes all versions up to the given one.
DeleteResponse deleteResponse =
client()
.prepareDelete("test", "type", "1")
.setVersion(17)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
assertThat(deleteResponse.getVersion(), equalTo(17L));
// Deleting with a lower version keeps on failing after a delete.
assertThrows(
client()
.prepareDelete("test", "type", "1")
.setVersion(2)
.setVersionType(VersionType.EXTERNAL)
.execute(),
VersionConflictEngineException.class);
// But delete with a higher version is OK.
deleteResponse =
client()
.prepareDelete("test", "type", "1")
.setVersion(18)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertEquals(DocWriteResponse.Operation.NOOP, deleteResponse.getOperation());
assertThat(deleteResponse.getVersion(), equalTo(18L));
// TODO: This behavior breaks rest api returning http status 201, good news is that it this is
// only the case until deletes GC kicks in.
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(19)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(19L));
deleteResponse =
client()
.prepareDelete("test", "type", "1")
.setVersion(20)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertEquals(DocWriteResponse.Operation.DELETE, deleteResponse.getOperation());
assertThat(deleteResponse.getVersion(), equalTo(20L));
// Make sure that the next delete will be GC. Note we do it on the index settings so it will be
// cleaned up
HashMap<String, Object> newSettings = new HashMap<>();
newSettings.put("index.gc_deletes", -1);
client()
.admin()
.indices()
.prepareUpdateSettings("test")
.setSettings(newSettings)
.execute()
.actionGet();
Thread.sleep(300); // gc works based on estimated sampled time. Give it a chance...
// And now we have previous version return -1
indexResponse =
client()
.prepareIndex("test", "type", "1")
.setSource("field1", "value1_1")
.setVersion(20)
.setVersionType(VersionType.EXTERNAL)
.execute()
.actionGet();
assertThat(indexResponse.getVersion(), equalTo(20L));
}
@Test
public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
final int numShards = between(1, 5);
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("test")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
final double exceptionRate;
final double exceptionOnOpenRate;
if (frequently()) {
if (randomBoolean()) {
if (randomBoolean()) {
exceptionOnOpenRate = 1.0 / between(5, 100);
exceptionRate = 0.0d;
} else {
exceptionRate = 1.0 / between(5, 100);
exceptionOnOpenRate = 0.0d;
}
} else {
exceptionOnOpenRate = 1.0 / between(5, 100);
exceptionRate = 1.0 / between(5, 100);
}
} else {
// rarely no exception
exceptionRate = 0d;
exceptionOnOpenRate = 0d;
}
Builder settings =
settingsBuilder()
.put("index.number_of_shards", numShards)
.put("index.number_of_replicas", randomIntBetween(0, 1))
.put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE, exceptionRate)
.put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE_ON_OPEN, exceptionOnOpenRate)
.put(MockDirectoryHelper.CHECK_INDEX_ON_CLOSE, true);
logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(settings)
.addMapping("type", mapping)
.execute()
.actionGet();
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.health(
Requests.clusterHealthRequest()
.waitForYellowStatus()
.timeout(TimeValue.timeValueSeconds(5)))
.get(); // it's OK to timeout here
final int numDocs;
final boolean expectAllShardsFailed;
if (clusterHealthResponse.isTimedOut()) {
/* some seeds just won't let you create the index at all and we enter a ping-pong mode
* trying one node after another etc. that is ok but we need to make sure we don't wait
* forever when indexing documents so we set numDocs = 1 and expecte all shards to fail
* when we search below.*/
logger.info("ClusterHealth timed out - only index one doc and expect searches to fail");
numDocs = 1;
expectAllShardsFailed = true;
} else {
numDocs = between(10, 100);
expectAllShardsFailed = false;
}
long numCreated = 0;
boolean[] added = new boolean[numDocs];
for (int i = 0; i < numDocs; i++) {
try {
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "" + i)
.setTimeout(TimeValue.timeValueSeconds(1))
.setSource("test", English.intToEnglish(i))
.get();
if (indexResponse.isCreated()) {
numCreated++;
added[i] = true;
}
} catch (ElasticSearchException ex) {
}
}
logger.info("Start Refresh");
RefreshResponse refreshResponse =
client()
.admin()
.indices()
.prepareRefresh("test")
.execute()
.get(); // don't assert on failures here
final boolean refreshFailed =
refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
logger.info(
"Refresh failed [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ",
refreshFailed,
refreshResponse.getFailedShards(),
refreshResponse.getShardFailures().length,
refreshResponse.getSuccessfulShards(),
refreshResponse.getTotalShards());
final int numSearches = atLeast(10);
// we don't check anything here really just making sure we don't leave any open files or a
// broken index behind.
for (int i = 0; i < numSearches; i++) {
try {
int docToQuery = between(0, numDocs - 1);
long expectedResults = added[docToQuery] ? 1 : 0;
logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery)))
.get();
logger.info(
"Successful shards: [{}] numShards: [{}]",
searchResponse.getSuccessfulShards(),
numShards);
if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(expectedResults));
}
// check match all
searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).get();
if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(numCreated));
}
} catch (SearchPhaseExecutionException ex) {
if (!expectAllShardsFailed) {
throw ex;
} else {
logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
}
}
}
}
@Test
public void deleteTest() throws ExecutionException, InterruptedException {
final String target = "test-target";
final String idx = "test-index";
final String tp = "test-type";
final String id = "1234";
final String othertp = "othertest-type";
// create target index if it doesn't exist
if (!ln.es.admin().indices().prepareExists(target).execute().actionGet().exists()) {
final CreateIndexResponse cir =
ln.es.admin().indices().prepareCreate(target).execute().actionGet();
if (!cir.acknowledged()) throw new RuntimeException("failed to create index " + target);
}
final IndexResponse ir =
ln.es.prepareIndex(idx, tp, id).setSource("{\"children\":[1,2,3]}").execute().actionGet();
ln.log.info(
"Indexed /{}/{}/{} as version {} into ES", new Object[] {idx, tp, id, ir.version()});
final IndexResponse ir1 =
ln.es
.prepareIndex(idx, tp, "1")
.setSource("{\"cheese\":\"gorgonzola\"}")
.execute()
.actionGet();
ln.log.info("Indexed /{}/{}/1 as version {} into ES", new Object[] {idx, tp, ir1.version()});
final IndexResponse ir2 =
ln.es
.prepareIndex(idx, tp, "2")
.setSource("{\"cheese\":\"mozarella\"}")
.execute()
.actionGet();
ln.log.info("Indexed /{}/{}/2 as version {} into ES", new Object[] {idx, tp, ir2.version()});
final IndexResponse ir3 =
ln.es
.prepareIndex(idx, othertp, "3")
.setSource("{\"cheese\":\"limburger\"}")
.execute()
.actionGet();
ln.log.info(
"Indexed /{}/{}/3 as version {} into ES", new Object[] {idx, othertp, ir3.version()});
final List<Future<DegraphmalizeResult>> actions = new ArrayList<Future<DegraphmalizeResult>>();
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.UPDATE,
DegraphmalizeRequestScope.DOCUMENT,
new ID(idx, tp, id, ir.version()),
ln.callback));
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.UPDATE,
DegraphmalizeRequestScope.DOCUMENT,
new ID(idx, tp, "1", ir1.version()),
ln.callback));
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.UPDATE,
DegraphmalizeRequestScope.DOCUMENT,
new ID(idx, tp, "2", ir2.version()),
ln.callback));
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.UPDATE,
DegraphmalizeRequestScope.DOCUMENT,
new ID(idx, othertp, "3", ir3.version()),
ln.callback));
for (final Future<DegraphmalizeResult> a : actions) {
DegraphmalizeResult result = a.get();
ln.log.info("Degraphmalize complete for : " + result.root());
}
GraphUtilities.dumpGraph(new ObjectMapper(), ln.G);
actions.clear();
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.DELETE,
DegraphmalizeRequestScope.DOCUMENT_ANY_VERSION,
new ID(idx, tp, id, 0),
ln.callback));
for (final Future<DegraphmalizeResult> a : actions) {
DegraphmalizeResult result = a.get();
ln.log.info("Degraphmalize of {}: {}", result.root(), result);
}
actions.clear();
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.DELETE,
DegraphmalizeRequestScope.TYPE_IN_INDEX,
new ID(idx, tp, null, 0),
ln.callback));
for (final Future<DegraphmalizeResult> a : actions) {
DegraphmalizeResult result = a.get();
ln.log.info("Degraphmalize of {}: {}", result.root(), result);
}
// Only the vertex of the othertp type should be present.
Iterable<Vertex> iterable = GraphUtilities.findVerticesInIndex(ln.G, idx);
assertThat(Iterables.size(iterable)).isEqualTo(1);
actions.clear();
actions.add(
ln.d.degraphmalize(
DegraphmalizeRequestType.DELETE,
DegraphmalizeRequestScope.INDEX,
new ID(idx, null, null, 0),
ln.callback));
for (final Future<DegraphmalizeResult> a : actions) {
DegraphmalizeResult result = a.get();
ln.log.info("Degraphmalize of {}: {}", result.root(), result);
}
// No more vertices
Iterable<Vertex> iterable2 = GraphUtilities.findVerticesInIndex(ln.G, idx);
assertThat(Iterables.size(iterable2)).isEqualTo(0);
// Cleanup index
if (!ln.es.admin().indices().delete(new DeleteIndexRequest(idx)).actionGet().acknowledged()) {
throw new RuntimeException("failed to delete index " + target);
}
}
@Override
protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(
ClusterState clusterState, PrimaryOperationRequest shardRequest) {
final BulkShardRequest request = shardRequest.request;
final IndexService indexService = indicesService.indexServiceSafe(request.index());
final IndexShard indexShard = indexService.shardSafe(shardRequest.shardId.id());
long[] preVersions = new long[request.items().length];
VersionType[] preVersionTypes = new VersionType[request.items().length];
Translog.Location location = null;
for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
BulkItemRequest item = request.items()[requestIndex];
if (item.request() instanceof IndexRequest) {
IndexRequest indexRequest = (IndexRequest) item.request();
preVersions[requestIndex] = indexRequest.version();
preVersionTypes[requestIndex] = indexRequest.versionType();
try {
WriteResult<IndexResponse> result =
shardIndexOperation(request, indexRequest, clusterState, indexShard, true);
location = locationToSync(location, result.location);
// add the response
IndexResponse indexResponse = result.response();
setResponse(
item,
new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < requestIndex; j++) {
applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
}
throw (ElasticsearchException) e;
}
if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
} else {
logger.debug(
"{} failed to execute bulk item (index) {}", e, shardRequest.shardId, indexRequest);
}
// if its a conflict failure, and we already executed the request on a primary (and we
// execute it
// again, due to primary relocation and only processing up to N bulk items when the shard
// gets closed)
// then just use the response we got from the successful execution
if (item.getPrimaryResponse() != null && isConflictException(e)) {
setResponse(item, item.getPrimaryResponse());
} else {
setResponse(
item,
new BulkItemResponse(
item.id(),
indexRequest.opType().lowercase(),
new BulkItemResponse.Failure(
request.index(), indexRequest.type(), indexRequest.id(), e)));
}
}
} else if (item.request() instanceof DeleteRequest) {
DeleteRequest deleteRequest = (DeleteRequest) item.request();
preVersions[requestIndex] = deleteRequest.version();
preVersionTypes[requestIndex] = deleteRequest.versionType();
try {
// add the response
final WriteResult<DeleteResponse> writeResult =
shardDeleteOperation(request, deleteRequest, indexShard);
DeleteResponse deleteResponse = writeResult.response();
location = locationToSync(location, writeResult.location);
setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
} catch (Throwable e) {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(e)) {
// restore updated versions...
for (int j = 0; j < requestIndex; j++) {
applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
}
throw (ElasticsearchException) e;
}
if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (delete) {}",
e,
shardRequest.shardId,
deleteRequest);
} else {
logger.debug(
"{} failed to execute bulk item (delete) {}",
e,
shardRequest.shardId,
deleteRequest);
}
// if its a conflict failure, and we already executed the request on a primary (and we
// execute it
// again, due to primary relocation and only processing up to N bulk items when the shard
// gets closed)
// then just use the response we got from the successful execution
if (item.getPrimaryResponse() != null && isConflictException(e)) {
setResponse(item, item.getPrimaryResponse());
} else {
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_DELETE,
new BulkItemResponse.Failure(
request.index(), deleteRequest.type(), deleteRequest.id(), e)));
}
}
} else if (item.request() instanceof UpdateRequest) {
UpdateRequest updateRequest = (UpdateRequest) item.request();
preVersions[requestIndex] = updateRequest.version();
preVersionTypes[requestIndex] = updateRequest.versionType();
// We need to do the requested retries plus the initial attempt. We don't do <
// 1+retry_on_conflict because retry_on_conflict may be Integer.MAX_VALUE
for (int updateAttemptsCount = 0;
updateAttemptsCount <= updateRequest.retryOnConflict();
updateAttemptsCount++) {
UpdateResult updateResult;
try {
updateResult = shardUpdateOperation(clusterState, request, updateRequest, indexShard);
} catch (Throwable t) {
updateResult = new UpdateResult(null, null, false, t, null);
}
if (updateResult.success()) {
if (updateResult.writeResult != null) {
location = locationToSync(location, updateResult.writeResult.location);
}
switch (updateResult.result.operation()) {
case UPSERT:
case INDEX:
WriteResult<IndexResponse> result = updateResult.writeResult;
IndexRequest indexRequest = updateResult.request();
BytesReference indexSourceAsBytes = indexRequest.source();
// add the response
IndexResponse indexResponse = result.response();
UpdateResponse updateResponse =
new UpdateResponse(
indexResponse.getShardInfo(),
indexResponse.getIndex(),
indexResponse.getType(),
indexResponse.getId(),
indexResponse.getVersion(),
indexResponse.isCreated());
if (updateRequest.fields() != null && updateRequest.fields().length > 0) {
Tuple<XContentType, Map<String, Object>> sourceAndContent =
XContentHelper.convertToMap(indexSourceAsBytes, true);
updateResponse.setGetResult(
updateHelper.extractGetResult(
updateRequest,
shardRequest.request.index(),
indexResponse.getVersion(),
sourceAndContent.v2(),
sourceAndContent.v1(),
indexSourceAsBytes));
}
item =
request.items()[requestIndex] =
new BulkItemRequest(request.items()[requestIndex].id(), indexRequest);
setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
break;
case DELETE:
WriteResult<DeleteResponse> writeResult = updateResult.writeResult;
DeleteResponse response = writeResult.response();
DeleteRequest deleteRequest = updateResult.request();
updateResponse =
new UpdateResponse(
response.getShardInfo(),
response.getIndex(),
response.getType(),
response.getId(),
response.getVersion(),
false);
updateResponse.setGetResult(
updateHelper.extractGetResult(
updateRequest,
shardRequest.request.index(),
response.getVersion(),
updateResult.result.updatedSourceAsMap(),
updateResult.result.updateSourceContentType(),
null));
// Replace the update request to the translated delete request to execute on the
// replica.
item =
request.items()[requestIndex] =
new BulkItemRequest(request.items()[requestIndex].id(), deleteRequest);
setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
break;
case NONE:
setResponse(
item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResult.noopResult));
item.setIgnoreOnReplica(); // no need to go to the replica
break;
}
// NOTE: Breaking out of the retry_on_conflict loop!
break;
} else if (updateResult.failure()) {
Throwable t = updateResult.error;
if (updateResult.retry) {
// updateAttemptCount is 0 based and marks current attempt, if it's equal to
// retryOnConflict we are going out of the iteration
if (updateAttemptsCount >= updateRequest.retryOnConflict()) {
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_UPDATE,
new BulkItemResponse.Failure(
request.index(), updateRequest.type(), updateRequest.id(), t)));
}
} else {
// rethrow the failure if we are going to retry on primary and let parent failure to
// handle it
if (retryPrimaryException(t)) {
// restore updated versions...
for (int j = 0; j < requestIndex; j++) {
applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
}
throw (ElasticsearchException) t;
}
// if its a conflict failure, and we already executed the request on a primary (and we
// execute it
// again, due to primary relocation and only processing up to N bulk items when the
// shard gets closed)
// then just use the response we got from the successful execution
if (item.getPrimaryResponse() != null && isConflictException(t)) {
setResponse(item, item.getPrimaryResponse());
} else if (updateResult.result == null) {
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_UPDATE,
new BulkItemResponse.Failure(
shardRequest.request.index(),
updateRequest.type(),
updateRequest.id(),
t)));
} else {
switch (updateResult.result.operation()) {
case UPSERT:
case INDEX:
IndexRequest indexRequest = updateResult.request();
if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (index) {}",
t,
shardRequest.shardId,
indexRequest);
} else {
logger.debug(
"{} failed to execute bulk item (index) {}",
t,
shardRequest.shardId,
indexRequest);
}
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_UPDATE,
new BulkItemResponse.Failure(
request.index(), indexRequest.type(), indexRequest.id(), t)));
break;
case DELETE:
DeleteRequest deleteRequest = updateResult.request();
if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
logger.trace(
"{} failed to execute bulk item (delete) {}",
t,
shardRequest.shardId,
deleteRequest);
} else {
logger.debug(
"{} failed to execute bulk item (delete) {}",
t,
shardRequest.shardId,
deleteRequest);
}
setResponse(
item,
new BulkItemResponse(
item.id(),
OP_TYPE_DELETE,
new BulkItemResponse.Failure(
request.index(), deleteRequest.type(), deleteRequest.id(), t)));
break;
}
}
// NOTE: Breaking out of the retry_on_conflict loop!
break;
}
}
}
} else {
throw new IllegalStateException("Unexpected index operation: " + item.request());
}
assert item.getPrimaryResponse() != null;
assert preVersionTypes[requestIndex] != null;
}
processAfter(request.refresh(), indexShard, location);
BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
BulkItemRequest[] items = request.items();
for (int i = 0; i < items.length; i++) {
responses[i] = items[i].getPrimaryResponse();
}
return new Tuple<>(
new BulkShardResponse(shardRequest.shardId, responses), shardRequest.request);
}