public void testIndexNameDateMathExpressions() {
DateTime now = new DateTime(DateTimeZone.UTC);
String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now);
String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1));
String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2));
createIndex(index1, index2, index3);
String dateMathExp1 = "<.marvel-{now/d}>";
String dateMathExp2 = "<.marvel-{now/d-1d}>";
String dateMathExp3 = "<.marvel-{now/d-2d}>";
client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
refresh();
SearchResponse searchResponse =
client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
assertHitCount(searchResponse, 3);
assertSearchHits(searchResponse, "1", "2", "3");
GetResponse getResponse = client().prepareGet(dateMathExp1, "type", "1").get();
assertThat(getResponse.isExists(), is(true));
assertThat(getResponse.getId(), equalTo("1"));
getResponse = client().prepareGet(dateMathExp2, "type", "2").get();
assertThat(getResponse.isExists(), is(true));
assertThat(getResponse.getId(), equalTo("2"));
getResponse = client().prepareGet(dateMathExp3, "type", "3").get();
assertThat(getResponse.isExists(), is(true));
assertThat(getResponse.getId(), equalTo("3"));
IndicesStatsResponse indicesStatsResponse =
client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get();
assertThat(indicesStatsResponse.getIndex(index1), notNullValue());
assertThat(indicesStatsResponse.getIndex(index2), notNullValue());
assertThat(indicesStatsResponse.getIndex(index3), notNullValue());
DeleteResponse deleteResponse = client().prepareDelete(dateMathExp1, "type", "1").get();
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
assertThat(deleteResponse.getId(), equalTo("1"));
deleteResponse = client().prepareDelete(dateMathExp2, "type", "2").get();
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
assertThat(deleteResponse.getId(), equalTo("2"));
deleteResponse = client().prepareDelete(dateMathExp3, "type", "3").get();
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
assertThat(deleteResponse.getId(), equalTo("3"));
}
@Override
public void run() {
long totalRefreshTime = 0;
int numExecutedRefreshed = 0;
while (run) {
long docIdLimit = COUNT;
for (long docId = 1; run && docId < docIdLimit; ) {
try {
for (int j = 0; j < 8; j++) {
GetResponse getResponse =
client.prepareGet(indexName, "type1", String.valueOf(++docId)).get();
client
.prepareIndex(indexName, "type1", getResponse.getId())
.setSource(getResponse.getSource())
.get();
}
long startTime = System.currentTimeMillis();
client.admin().indices().prepareRefresh(indexName).execute().actionGet();
totalRefreshTime += System.currentTimeMillis() - startTime;
numExecutedRefreshed++;
Thread.sleep(500);
} catch (Throwable e) {
e.printStackTrace();
}
}
}
avgRefreshTime = totalRefreshTime / numExecutedRefreshed;
stopped = true;
}
@Override
public <T> T mapResult(GetResponse response, Class<T> clazz) {
T result = mapEntity(response.getSourceAsString(), clazz);
if (result != null) {
setPersistentEntityId(result, response.getId(), clazz);
}
return result;
}
@Test
public void testGet() {
GetResponse response =
client
.prepareGet("asdf2014", "asdf", "1")
.setOperationThreaded(false)
.execute()
.actionGet();
if (response != null) System.out.println("Id: " + response.getId());
}
public T get(String id) {
GetResponse result = client.prepareGet(index, entity, id).execute().actionGet();
String source = result.getSourceAsString();
System.out.println("Loaded SLA: " + source);
try {
T entity = mapper.readValue(source, clazz);
entity.setId(result.getId());
logger.info("Successfully obtained entity with id [{}]", id);
return entity;
} catch (IOException e) {
logger.error("Error retrieving entity with id " + id, e);
return null;
}
}
@SuppressWarnings("unchecked")
protected <T extends ESEntity> T buildEntityFromGetResponse(
Class<T> entityClass, MultiGetItemResponse item) {
GetResponse response = item.getResponse();
if (!response.isExists())
throw new DaoException(
String.format(
"Entity %s does not exist in %s/%s",
response.getId(), response.getIndex(), response.getType()));
if (entityClass == null) throw new IllegalArgumentException("Provided Entity class is null");
else if (entityClass.equals(ESNode.class))
return (T) ESNode.Builder.buildFromGetReponse(response);
else if (entityClass.equals(ESWay.class))
return (T) ESWay.Builder.buildFromGetReponse(response);
else throw new IllegalArgumentException(entityClass.getSimpleName() + " is not a known Entity");
}
@Test
public void testGetSlash() {
final String id = "get-test/id-2";
final IndexRequestBuilder indexRequestBuilder =
restClient().prepareIndex(index, type, id).setSource("field", "value").setRefresh(true);
indexRequestBuilder.execute().actionGet();
final GetRequestBuilder getRequestBuilder = restClient().prepareGet(index, type, id);
final ListenableActionFuture<GetResponse> execute1 = getRequestBuilder.execute();
final GetResponse get = execute1.actionGet();
assertEquals(get.getIndex(), index);
assertEquals(get.getType(), type);
assertEquals(get.getId(), id);
assertEquals(get.getVersion(), 1);
assertTrue(get.getFields().isEmpty());
assertEquals(get.getSource().get("field"), "value");
}
@Override
public Article get(long id) {
GetResponse getResponse =
transportClient.prepareGet(INDICE, TYPE, String.valueOf(id)).execute().actionGet();
if (getResponse.isExists()) {
Map<String, Object> map = getResponse.getSource();
String title = (String) map.get(TITLE);
String summary = (String) map.get(SUMMARY);
String idStr = getResponse.getId();
logger.debug("get id : {}", idStr);
Article article = new Article();
article.setId(Long.parseLong(idStr));
article.setTitle(title);
article.setSummary(summary);
return article;
} else {
return null;
}
}
public void testIndexGetAndDelete() throws ExecutionException, InterruptedException {
createIndexWithAlias();
ensureYellow("test");
int numDocs = iterations(10, 50);
for (int i = 0; i < numDocs; i++) {
IndexResponse indexResponse =
client()
.prepareIndex(indexOrAlias(), "type", Integer.toString(i))
.setSource("field", "value-" + i)
.get();
assertThat(indexResponse.isCreated(), equalTo(true));
assertThat(indexResponse.getIndex(), equalTo("test"));
assertThat(indexResponse.getType(), equalTo("type"));
assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
}
refresh();
String docId = Integer.toString(randomIntBetween(0, numDocs - 1));
GetResponse getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
assertThat(getResponse.isExists(), equalTo(true));
assertThat(getResponse.getIndex(), equalTo("test"));
assertThat(getResponse.getType(), equalTo("type"));
assertThat(getResponse.getId(), equalTo(docId));
DeleteResponse deleteResponse = client().prepareDelete(indexOrAlias(), "type", docId).get();
assertThat(deleteResponse.isFound(), equalTo(true));
assertThat(deleteResponse.getIndex(), equalTo("test"));
assertThat(deleteResponse.getType(), equalTo("type"));
assertThat(deleteResponse.getId(), equalTo(docId));
getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
assertThat(getResponse.isExists(), equalTo(false));
refresh();
SearchResponse searchResponse = client().prepareSearch(indexOrAlias()).get();
assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1));
}
public final ModelT getModelById(
final ModelIdT id,
final Logger logger,
final Marker logMarker,
final ModelFactory<ModelEntryT> modelFactory)
throws InvalidModelException, IoExceptionT, NoSuchModelExceptionT {
logger.debug(logMarker, "getting model {} from index {}", id, indexName);
try {
final GetResponse response =
client.prepareGet(indexName, documentType, id.toString()).execute().actionGet();
if (!response.isExists()) {
throw exceptionFactory.newNoSuchModelException(id);
}
return modelFactory
.createModelEntry(response.getId(), response.getSourceAsBytesRef())
.getModel();
} catch (final IndexNotFoundException e) {
logger.warn(logMarker, "tried to get model {} from missing index {}", id, indexName);
throw exceptionFactory.newNoSuchModelException(id);
} catch (final ElasticsearchException e) {
throw exceptionFactory.newIoException(
e, String.format("error getting model %s from index %s", id, indexName));
}
}
@Test
public void testGetDocWithMultivaluedFields() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// fine
}
String mapping1 =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("field")
.field("type", "string")
.field("store", "yes")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
String mapping2 =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type2")
.startObject("properties")
.startObject("field")
.field("type", "string")
.field("store", "yes")
.endObject()
.endObject()
.startObject("_source")
.field("enabled", false)
.endObject()
.endObject()
.endObject()
.string();
client
.admin()
.indices()
.prepareCreate("test")
.addMapping("type1", mapping1)
.addMapping("type2", mapping2)
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
.execute()
.actionGet();
ClusterHealthResponse clusterHealth =
client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
GetResponse response = client.prepareGet("test", "type1", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(false));
response = client.prepareGet("test", "type2", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(false));
client
.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
.execute()
.actionGet();
client
.prepareIndex("test", "type2", "1")
.setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
.execute()
.actionGet();
response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getId(), equalTo("1"));
assertThat(response.getType(), equalTo("type1"));
assertThat(response.getFields().size(), equalTo(1));
assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
equalTo("1"));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
equalTo("2"));
response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getType(), equalTo("type2"));
assertThat(response.getId(), equalTo("1"));
assertThat(response.getFields().size(), equalTo(1));
assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
equalTo("1"));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
equalTo("2"));
// Now test values being fetched from stored fields.
client.admin().indices().prepareRefresh("test").execute().actionGet();
response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getId(), equalTo("1"));
assertThat(response.getFields().size(), equalTo(1));
assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
equalTo("1"));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
equalTo("2"));
response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getId(), equalTo("1"));
assertThat(response.getFields().size(), equalTo(1));
assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
equalTo("1"));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
equalTo("2"));
}
@Test
@Slow
public void stressUpdateDeleteConcurrency() throws Exception {
// We create an index with merging disabled so that deletes don't get merged away
client()
.admin()
.indices()
.prepareCreate("test")
.addMapping(
"type1",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.field("store", "yes")
.endObject()
.endObject()
.endObject())
.setSettings(
ImmutableSettings.builder()
.put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, NoMergePolicyProvider.class))
.execute()
.actionGet();
ensureGreen();
final int numberOfThreads = scaledRandomIntBetween(5, 10);
final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
final int numberOfUpdatesPerId = scaledRandomIntBetween(100, 200);
final int retryOnConflict = randomIntBetween(0, 1);
final CountDownLatch latch = new CountDownLatch(numberOfThreads);
final CountDownLatch startLatch = new CountDownLatch(1);
final List<Throwable> failures = new CopyOnWriteArrayList<>();
final class UpdateThread extends Thread {
final Map<Integer, Integer> failedMap = new HashMap<>();
final int numberOfIds;
final int updatesPerId;
final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);
private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);
public UpdateThread(int numberOfIds, int updatesPerId) {
this.numberOfIds = numberOfIds;
this.updatesPerId = updatesPerId;
}
final class UpdateListener implements ActionListener<UpdateResponse> {
int id;
public UpdateListener(int id) {
this.id = id;
}
@Override
public void onResponse(UpdateResponse updateResponse) {
updateRequestsOutstanding.release(1);
}
@Override
public void onFailure(Throwable e) {
synchronized (failedMap) {
incrementMapValue(id, failedMap);
}
updateRequestsOutstanding.release(1);
}
}
final class DeleteListener implements ActionListener<DeleteResponse> {
int id;
public DeleteListener(int id) {
this.id = id;
}
@Override
public void onResponse(DeleteResponse deleteResponse) {
deleteRequestsOutstanding.release(1);
}
@Override
public void onFailure(Throwable e) {
synchronized (failedMap) {
incrementMapValue(id, failedMap);
}
deleteRequestsOutstanding.release(1);
}
}
@Override
public void run() {
try {
startLatch.await();
for (int j = 0; j < numberOfIds; j++) {
for (int k = 0; k < numberOfUpdatesPerId; ++k) {
updateRequestsOutstanding.acquire();
UpdateRequest ur =
client()
.prepareUpdate("test", "type1", Integer.toString(j))
.setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
.setRetryOnConflict(retryOnConflict)
.setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
.setListenerThreaded(false)
.request();
client().update(ur, new UpdateListener(j));
deleteRequestsOutstanding.acquire();
DeleteRequest dr =
client()
.prepareDelete("test", "type1", Integer.toString(j))
.setListenerThreaded(false)
.setOperationThreaded(false)
.request();
client().delete(dr, new DeleteListener(j));
}
}
} catch (Throwable e) {
logger.error("Something went wrong", e);
failures.add(e);
} finally {
try {
waitForOutstandingRequests(
TimeValue.timeValueSeconds(60),
updateRequestsOutstanding,
maxUpdateRequests,
"Update");
waitForOutstandingRequests(
TimeValue.timeValueSeconds(60),
deleteRequestsOutstanding,
maxDeleteRequests,
"Delete");
} catch (ElasticsearchTimeoutException ete) {
failures.add(ete);
}
latch.countDown();
}
}
private void incrementMapValue(int j, Map<Integer, Integer> map) {
if (!map.containsKey(j)) {
map.put(j, 0);
}
map.put(j, map.get(j) + 1);
}
private void waitForOutstandingRequests(
TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
long start = System.currentTimeMillis();
do {
long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
logger.info(
"[{}] going to try and aquire [{}] in [{}]ms [{}] available to aquire right now",
name,
maxRequests,
msRemaining,
requestsOutstanding.availablePermits());
try {
requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
return;
} catch (InterruptedException ie) {
// Just keep swimming
}
} while ((System.currentTimeMillis() - start) < timeOut.getMillis());
throw new ElasticsearchTimeoutException(
"Requests were still outstanding after the timeout ["
+ timeOut
+ "] for type ["
+ name
+ "]");
}
}
final List<UpdateThread> threads = new ArrayList<>();
for (int i = 0; i < numberOfThreads; i++) {
UpdateThread ut = new UpdateThread(numberOfIdsPerThread, numberOfUpdatesPerId);
ut.start();
threads.add(ut);
}
startLatch.countDown();
latch.await();
for (UpdateThread ut : threads) {
ut.join(); // Threads should have finished because of the latch.await
}
// If are no errors every request recieved a response otherwise the test would have timedout
// aquiring the request outstanding semaphores.
for (Throwable throwable : failures) {
logger.info("Captured failure on concurrent update:", throwable);
}
assertThat(failures.size(), equalTo(0));
// Upsert all the ids one last time to make sure they are available at get time
// This means that we add 1 to the expected versions and attempts
// All the previous operations should be complete or failed at this point
for (int i = 0; i < numberOfIdsPerThread; ++i) {
UpdateResponse ur =
client()
.prepareUpdate("test", "type1", Integer.toString(i))
.setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
.setRetryOnConflict(Integer.MAX_VALUE)
.setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
.execute()
.actionGet();
}
refresh();
for (int i = 0; i < numberOfIdsPerThread; ++i) {
int totalFailures = 0;
GetResponse response =
client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
if (response.isExists()) {
assertThat(response.getId(), equalTo(Integer.toString(i)));
int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
for (UpdateThread ut : threads) {
if (ut.failedMap.containsKey(i)) {
totalFailures += ut.failedMap.get(i);
}
}
expectedVersion -= totalFailures;
logger.error(
"Actual version [{}] Expected version [{}] Total failures [{}]",
response.getVersion(),
expectedVersion,
totalFailures);
assertThat(response.getVersion(), equalTo((long) expectedVersion));
assertThat(
response.getVersion() + totalFailures,
equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
}
}
}
@Test
@Slow
public void testConcurrentUpdateWithRetryOnConflict() throws Exception {
final boolean useBulkApi = randomBoolean();
createIndex();
ensureGreen();
int numberOfThreads = scaledRandomIntBetween(2, 5);
final CountDownLatch latch = new CountDownLatch(numberOfThreads);
final CountDownLatch startLatch = new CountDownLatch(1);
final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 10000);
final List<Throwable> failures = new CopyOnWriteArrayList<>();
for (int i = 0; i < numberOfThreads; i++) {
Runnable r =
new Runnable() {
@Override
public void run() {
try {
startLatch.await();
for (int i = 0; i < numberOfUpdatesPerThread; i++) {
if (useBulkApi) {
UpdateRequestBuilder updateRequestBuilder =
client()
.prepareUpdate("test", "type1", Integer.toString(i))
.setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
.setRetryOnConflict(Integer.MAX_VALUE)
.setUpsert(jsonBuilder().startObject().field("field", 1).endObject());
client().prepareBulk().add(updateRequestBuilder).execute().actionGet();
} else {
client()
.prepareUpdate("test", "type1", Integer.toString(i))
.setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE)
.setRetryOnConflict(Integer.MAX_VALUE)
.setUpsert(jsonBuilder().startObject().field("field", 1).endObject())
.execute()
.actionGet();
}
}
} catch (Throwable e) {
failures.add(e);
} finally {
latch.countDown();
}
}
};
new Thread(r).start();
}
startLatch.countDown();
latch.await();
for (Throwable throwable : failures) {
logger.info("Captured failure on concurrent update:", throwable);
}
assertThat(failures.size(), equalTo(0));
for (int i = 0; i < numberOfUpdatesPerThread; i++) {
GetResponse response =
client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
assertThat(response.getId(), equalTo(Integer.toString(i)));
assertThat(response.isExists(), equalTo(true));
assertThat(response.getVersion(), equalTo((long) numberOfThreads));
assertThat((Integer) response.getSource().get("field"), equalTo(numberOfThreads));
}
}
