public void testIndexNameDateMathExpressions() { DateTime now = new DateTime(DateTimeZone.UTC); String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now); String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1)); String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2)); createIndex(index1, index2, index3); String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; String dateMathExp3 = "<.marvel-{now/d-2d}>"; client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get(); client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get(); client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get(); refresh(); SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get(); assertHitCount(searchResponse, 3); assertSearchHits(searchResponse, "1", "2", "3"); GetResponse getResponse = client().prepareGet(dateMathExp1, "type", "1").get(); assertThat(getResponse.isExists(), is(true)); assertThat(getResponse.getId(), equalTo("1")); getResponse = client().prepareGet(dateMathExp2, "type", "2").get(); assertThat(getResponse.isExists(), is(true)); assertThat(getResponse.getId(), equalTo("2")); getResponse = client().prepareGet(dateMathExp3, "type", "3").get(); assertThat(getResponse.isExists(), is(true)); assertThat(getResponse.getId(), equalTo("3")); IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get(); assertThat(indicesStatsResponse.getIndex(index1), notNullValue()); assertThat(indicesStatsResponse.getIndex(index2), notNullValue()); assertThat(indicesStatsResponse.getIndex(index3), notNullValue()); DeleteResponse deleteResponse = client().prepareDelete(dateMathExp1, "type", "1").get(); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); assertThat(deleteResponse.getId(), equalTo("1")); deleteResponse = client().prepareDelete(dateMathExp2, "type", "2").get(); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); assertThat(deleteResponse.getId(), equalTo("2")); deleteResponse = client().prepareDelete(dateMathExp3, "type", "3").get(); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); assertThat(deleteResponse.getId(), equalTo("3")); }
@Override public void run() { long totalRefreshTime = 0; int numExecutedRefreshed = 0; while (run) { long docIdLimit = COUNT; for (long docId = 1; run && docId < docIdLimit; ) { try { for (int j = 0; j < 8; j++) { GetResponse getResponse = client.prepareGet(indexName, "type1", String.valueOf(++docId)).get(); client .prepareIndex(indexName, "type1", getResponse.getId()) .setSource(getResponse.getSource()) .get(); } long startTime = System.currentTimeMillis(); client.admin().indices().prepareRefresh(indexName).execute().actionGet(); totalRefreshTime += System.currentTimeMillis() - startTime; numExecutedRefreshed++; Thread.sleep(500); } catch (Throwable e) { e.printStackTrace(); } } } avgRefreshTime = totalRefreshTime / numExecutedRefreshed; stopped = true; }
@Override public <T> T mapResult(GetResponse response, Class<T> clazz) { T result = mapEntity(response.getSourceAsString(), clazz); if (result != null) { setPersistentEntityId(result, response.getId(), clazz); } return result; }
@Test public void testGet() { GetResponse response = client .prepareGet("asdf2014", "asdf", "1") .setOperationThreaded(false) .execute() .actionGet(); if (response != null) System.out.println("Id: " + response.getId()); }
public T get(String id) { GetResponse result = client.prepareGet(index, entity, id).execute().actionGet(); String source = result.getSourceAsString(); System.out.println("Loaded SLA: " + source); try { T entity = mapper.readValue(source, clazz); entity.setId(result.getId()); logger.info("Successfully obtained entity with id [{}]", id); return entity; } catch (IOException e) { logger.error("Error retrieving entity with id " + id, e); return null; } }
@SuppressWarnings("unchecked") protected <T extends ESEntity> T buildEntityFromGetResponse( Class<T> entityClass, MultiGetItemResponse item) { GetResponse response = item.getResponse(); if (!response.isExists()) throw new DaoException( String.format( "Entity %s does not exist in %s/%s", response.getId(), response.getIndex(), response.getType())); if (entityClass == null) throw new IllegalArgumentException("Provided Entity class is null"); else if (entityClass.equals(ESNode.class)) return (T) ESNode.Builder.buildFromGetReponse(response); else if (entityClass.equals(ESWay.class)) return (T) ESWay.Builder.buildFromGetReponse(response); else throw new IllegalArgumentException(entityClass.getSimpleName() + " is not a known Entity"); }
@Test public void testGetSlash() { final String id = "get-test/id-2"; final IndexRequestBuilder indexRequestBuilder = restClient().prepareIndex(index, type, id).setSource("field", "value").setRefresh(true); indexRequestBuilder.execute().actionGet(); final GetRequestBuilder getRequestBuilder = restClient().prepareGet(index, type, id); final ListenableActionFuture<GetResponse> execute1 = getRequestBuilder.execute(); final GetResponse get = execute1.actionGet(); assertEquals(get.getIndex(), index); assertEquals(get.getType(), type); assertEquals(get.getId(), id); assertEquals(get.getVersion(), 1); assertTrue(get.getFields().isEmpty()); assertEquals(get.getSource().get("field"), "value"); }
@Override public Article get(long id) { GetResponse getResponse = transportClient.prepareGet(INDICE, TYPE, String.valueOf(id)).execute().actionGet(); if (getResponse.isExists()) { Map<String, Object> map = getResponse.getSource(); String title = (String) map.get(TITLE); String summary = (String) map.get(SUMMARY); String idStr = getResponse.getId(); logger.debug("get id : {}", idStr); Article article = new Article(); article.setId(Long.parseLong(idStr)); article.setTitle(title); article.setSummary(summary); return article; } else { return null; } }
public void testIndexGetAndDelete() throws ExecutionException, InterruptedException { createIndexWithAlias(); ensureYellow("test"); int numDocs = iterations(10, 50); for (int i = 0; i < numDocs; i++) { IndexResponse indexResponse = client() .prepareIndex(indexOrAlias(), "type", Integer.toString(i)) .setSource("field", "value-" + i) .get(); assertThat(indexResponse.isCreated(), equalTo(true)); assertThat(indexResponse.getIndex(), equalTo("test")); assertThat(indexResponse.getType(), equalTo("type")); assertThat(indexResponse.getId(), equalTo(Integer.toString(i))); } refresh(); String docId = Integer.toString(randomIntBetween(0, numDocs - 1)); GetResponse getResponse = client().prepareGet(indexOrAlias(), "type", docId).get(); assertThat(getResponse.isExists(), equalTo(true)); assertThat(getResponse.getIndex(), equalTo("test")); assertThat(getResponse.getType(), equalTo("type")); assertThat(getResponse.getId(), equalTo(docId)); DeleteResponse deleteResponse = client().prepareDelete(indexOrAlias(), "type", docId).get(); assertThat(deleteResponse.isFound(), equalTo(true)); assertThat(deleteResponse.getIndex(), equalTo("test")); assertThat(deleteResponse.getType(), equalTo("type")); assertThat(deleteResponse.getId(), equalTo(docId)); getResponse = client().prepareGet(indexOrAlias(), "type", docId).get(); assertThat(getResponse.isExists(), equalTo(false)); refresh(); SearchResponse searchResponse = client().prepareSearch(indexOrAlias()).get(); assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1)); }
public final ModelT getModelById( final ModelIdT id, final Logger logger, final Marker logMarker, final ModelFactory<ModelEntryT> modelFactory) throws InvalidModelException, IoExceptionT, NoSuchModelExceptionT { logger.debug(logMarker, "getting model {} from index {}", id, indexName); try { final GetResponse response = client.prepareGet(indexName, documentType, id.toString()).execute().actionGet(); if (!response.isExists()) { throw exceptionFactory.newNoSuchModelException(id); } return modelFactory .createModelEntry(response.getId(), response.getSourceAsBytesRef()) .getModel(); } catch (final IndexNotFoundException e) { logger.warn(logMarker, "tried to get model {} from missing index {}", id, indexName); throw exceptionFactory.newNoSuchModelException(id); } catch (final ElasticsearchException e) { throw exceptionFactory.newIoException( e, String.format("error getting model %s from index %s", id, indexName)); } }
@Test public void testGetDocWithMultivaluedFields() throws Exception { try { client.admin().indices().prepareDelete("test").execute().actionGet(); } catch (Exception e) { // fine } String mapping1 = XContentFactory.jsonBuilder() .startObject() .startObject("type1") .startObject("properties") .startObject("field") .field("type", "string") .field("store", "yes") .endObject() .endObject() .endObject() .endObject() .string(); String mapping2 = XContentFactory.jsonBuilder() .startObject() .startObject("type2") .startObject("properties") .startObject("field") .field("type", "string") .field("store", "yes") .endObject() .endObject() .startObject("_source") .field("enabled", false) .endObject() .endObject() .endObject() .string(); client .admin() .indices() .prepareCreate("test") .addMapping("type1", mapping1) .addMapping("type2", mapping2) .setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1)) .execute() .actionGet(); ClusterHealthResponse clusterHealth = client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet(); assertThat(clusterHealth.isTimedOut(), equalTo(false)); assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN)); GetResponse response = client.prepareGet("test", "type1", "1").execute().actionGet(); assertThat(response.isExists(), equalTo(false)); response = client.prepareGet("test", "type2", "1").execute().actionGet(); assertThat(response.isExists(), equalTo(false)); client .prepareIndex("test", "type1", "1") .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject()) .execute() .actionGet(); client .prepareIndex("test", "type2", "1") .setSource(jsonBuilder().startObject().field("field", "1", "2").endObject()) .execute() .actionGet(); response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet(); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getType(), equalTo("type1")); assertThat(response.getFields().size(), equalTo(1)); assertThat(response.getFields().get("field").getValues().size(), equalTo(1)); assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2)); assertThat( ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(), equalTo("1")); assertThat( ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(), equalTo("2")); response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet(); assertThat(response.isExists(), equalTo(true)); assertThat(response.getType(), equalTo("type2")); assertThat(response.getId(), equalTo("1")); assertThat(response.getFields().size(), equalTo(1)); assertThat(response.getFields().get("field").getValues().size(), equalTo(1)); assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2)); assertThat( ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(), equalTo("1")); assertThat( ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(), equalTo("2")); // Now test values being fetched from stored fields. client.admin().indices().prepareRefresh("test").execute().actionGet(); response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet(); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getFields().size(), equalTo(1)); assertThat(response.getFields().get("field").getValues().size(), equalTo(1)); assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2)); assertThat( ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(), equalTo("1")); assertThat( ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(), equalTo("2")); response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet(); assertThat(response.isExists(), equalTo(true)); assertThat(response.getId(), equalTo("1")); assertThat(response.getFields().size(), equalTo(1)); assertThat(response.getFields().get("field").getValues().size(), equalTo(1)); assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2)); assertThat( ((List) response.getFields().get("field").getValues().get(0)).get(0).toString(), equalTo("1")); assertThat( ((List) response.getFields().get("field").getValues().get(0)).get(1).toString(), equalTo("2")); }
@Test @Slow public void stressUpdateDeleteConcurrency() throws Exception { // We create an index with merging disabled so that deletes don't get merged away client() .admin() .indices() .prepareCreate("test") .addMapping( "type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") .startObject("_timestamp") .field("enabled", true) .field("store", "yes") .endObject() .startObject("_ttl") .field("enabled", true) .field("store", "yes") .endObject() .endObject() .endObject()) .setSettings( ImmutableSettings.builder() .put(MergePolicyModule.MERGE_POLICY_TYPE_KEY, NoMergePolicyProvider.class)) .execute() .actionGet(); ensureGreen(); final int numberOfThreads = scaledRandomIntBetween(5, 10); final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10); final int numberOfUpdatesPerId = scaledRandomIntBetween(100, 200); final int retryOnConflict = randomIntBetween(0, 1); final CountDownLatch latch = new CountDownLatch(numberOfThreads); final CountDownLatch startLatch = new CountDownLatch(1); final List<Throwable> failures = new CopyOnWriteArrayList<>(); final class UpdateThread extends Thread { final Map<Integer, Integer> failedMap = new HashMap<>(); final int numberOfIds; final int updatesPerId; final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId; final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId; private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests); private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests); public UpdateThread(int numberOfIds, int updatesPerId) { this.numberOfIds = numberOfIds; this.updatesPerId = updatesPerId; } final class UpdateListener implements ActionListener<UpdateResponse> { int id; public UpdateListener(int id) { this.id = id; } @Override public void onResponse(UpdateResponse updateResponse) { updateRequestsOutstanding.release(1); } @Override public void onFailure(Throwable e) { synchronized (failedMap) { incrementMapValue(id, failedMap); } updateRequestsOutstanding.release(1); } } final class DeleteListener implements ActionListener<DeleteResponse> { int id; public DeleteListener(int id) { this.id = id; } @Override public void onResponse(DeleteResponse deleteResponse) { deleteRequestsOutstanding.release(1); } @Override public void onFailure(Throwable e) { synchronized (failedMap) { incrementMapValue(id, failedMap); } deleteRequestsOutstanding.release(1); } } @Override public void run() { try { startLatch.await(); for (int j = 0; j < numberOfIds; j++) { for (int k = 0; k < numberOfUpdatesPerId; ++k) { updateRequestsOutstanding.acquire(); UpdateRequest ur = client() .prepareUpdate("test", "type1", Integer.toString(j)) .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE) .setRetryOnConflict(retryOnConflict) .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()) .setListenerThreaded(false) .request(); client().update(ur, new UpdateListener(j)); deleteRequestsOutstanding.acquire(); DeleteRequest dr = client() .prepareDelete("test", "type1", Integer.toString(j)) .setListenerThreaded(false) .setOperationThreaded(false) .request(); client().delete(dr, new DeleteListener(j)); } } } catch (Throwable e) { logger.error("Something went wrong", e); failures.add(e); } finally { try { waitForOutstandingRequests( TimeValue.timeValueSeconds(60), updateRequestsOutstanding, maxUpdateRequests, "Update"); waitForOutstandingRequests( TimeValue.timeValueSeconds(60), deleteRequestsOutstanding, maxDeleteRequests, "Delete"); } catch (ElasticsearchTimeoutException ete) { failures.add(ete); } latch.countDown(); } } private void incrementMapValue(int j, Map<Integer, Integer> map) { if (!map.containsKey(j)) { map.put(j, 0); } map.put(j, map.get(j) + 1); } private void waitForOutstandingRequests( TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) { long start = System.currentTimeMillis(); do { long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start); logger.info( "[{}] going to try and aquire [{}] in [{}]ms [{}] available to aquire right now", name, maxRequests, msRemaining, requestsOutstanding.availablePermits()); try { requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS); return; } catch (InterruptedException ie) { // Just keep swimming } } while ((System.currentTimeMillis() - start) < timeOut.getMillis()); throw new ElasticsearchTimeoutException( "Requests were still outstanding after the timeout [" + timeOut + "] for type [" + name + "]"); } } final List<UpdateThread> threads = new ArrayList<>(); for (int i = 0; i < numberOfThreads; i++) { UpdateThread ut = new UpdateThread(numberOfIdsPerThread, numberOfUpdatesPerId); ut.start(); threads.add(ut); } startLatch.countDown(); latch.await(); for (UpdateThread ut : threads) { ut.join(); // Threads should have finished because of the latch.await } // If are no errors every request recieved a response otherwise the test would have timedout // aquiring the request outstanding semaphores. for (Throwable throwable : failures) { logger.info("Captured failure on concurrent update:", throwable); } assertThat(failures.size(), equalTo(0)); // Upsert all the ids one last time to make sure they are available at get time // This means that we add 1 to the expected versions and attempts // All the previous operations should be complete or failed at this point for (int i = 0; i < numberOfIdsPerThread; ++i) { UpdateResponse ur = client() .prepareUpdate("test", "type1", Integer.toString(i)) .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE) .setRetryOnConflict(Integer.MAX_VALUE) .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()) .execute() .actionGet(); } refresh(); for (int i = 0; i < numberOfIdsPerThread; ++i) { int totalFailures = 0; GetResponse response = client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet(); if (response.isExists()) { assertThat(response.getId(), equalTo(Integer.toString(i))); int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1; for (UpdateThread ut : threads) { if (ut.failedMap.containsKey(i)) { totalFailures += ut.failedMap.get(i); } } expectedVersion -= totalFailures; logger.error( "Actual version [{}] Expected version [{}] Total failures [{}]", response.getVersion(), expectedVersion, totalFailures); assertThat(response.getVersion(), equalTo((long) expectedVersion)); assertThat( response.getVersion() + totalFailures, equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1))); } } }
@Test @Slow public void testConcurrentUpdateWithRetryOnConflict() throws Exception { final boolean useBulkApi = randomBoolean(); createIndex(); ensureGreen(); int numberOfThreads = scaledRandomIntBetween(2, 5); final CountDownLatch latch = new CountDownLatch(numberOfThreads); final CountDownLatch startLatch = new CountDownLatch(1); final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 10000); final List<Throwable> failures = new CopyOnWriteArrayList<>(); for (int i = 0; i < numberOfThreads; i++) { Runnable r = new Runnable() { @Override public void run() { try { startLatch.await(); for (int i = 0; i < numberOfUpdatesPerThread; i++) { if (useBulkApi) { UpdateRequestBuilder updateRequestBuilder = client() .prepareUpdate("test", "type1", Integer.toString(i)) .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE) .setRetryOnConflict(Integer.MAX_VALUE) .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()); client().prepareBulk().add(updateRequestBuilder).execute().actionGet(); } else { client() .prepareUpdate("test", "type1", Integer.toString(i)) .setScript("ctx._source.field += 1", ScriptService.ScriptType.INLINE) .setRetryOnConflict(Integer.MAX_VALUE) .setUpsert(jsonBuilder().startObject().field("field", 1).endObject()) .execute() .actionGet(); } } } catch (Throwable e) { failures.add(e); } finally { latch.countDown(); } } }; new Thread(r).start(); } startLatch.countDown(); latch.await(); for (Throwable throwable : failures) { logger.info("Captured failure on concurrent update:", throwable); } assertThat(failures.size(), equalTo(0)); for (int i = 0; i < numberOfUpdatesPerThread; i++) { GetResponse response = client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet(); assertThat(response.getId(), equalTo(Integer.toString(i))); assertThat(response.isExists(), equalTo(true)); assertThat(response.getVersion(), equalTo((long) numberOfThreads)); assertThat((Integer) response.getSource().get("field"), equalTo(numberOfThreads)); } }