public void testUpdate() {
createIndexWithAlias();
ensureYellow("test");
UpdateRequestBuilder updateRequestBuilder =
client()
.prepareUpdate(indexOrAlias(), "type1", "1")
.setUpsert("field1", "value1")
.setDoc("field2", "value2");
UpdateResponse updateResponse = updateRequestBuilder.get();
assertThat(updateResponse.getIndex(), equalTo("test"));
assertThat(updateResponse.getType(), equalTo("type1"));
assertThat(updateResponse.getId(), equalTo("1"));
assertThat(updateResponse.isCreated(), equalTo(true));
GetResponse getResponse = client().prepareGet("test", "type1", "1").get();
assertThat(getResponse.isExists(), equalTo(true));
assertThat(getResponse.getSourceAsMap().containsKey("field1"), equalTo(true));
assertThat(getResponse.getSourceAsMap().containsKey("field2"), equalTo(false));
updateResponse = updateRequestBuilder.get();
assertThat(updateResponse.getIndex(), equalTo("test"));
assertThat(updateResponse.getType(), equalTo("type1"));
assertThat(updateResponse.getId(), equalTo("1"));
assertThat(updateResponse.isCreated(), equalTo(false));
getResponse = client().prepareGet("test", "type1", "1").get();
assertThat(getResponse.isExists(), equalTo(true));
assertThat(getResponse.getSourceAsMap().containsKey("field1"), equalTo(true));
assertThat(getResponse.getSourceAsMap().containsKey("field2"), equalTo(true));
}
@Test
public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
String index = "test";
String type = "type1";
String mapping =
jsonBuilder()
.startObject()
.startObject("source_excludes")
.startObject("_source")
.array("includes", "included")
.array("exlcudes", "excluded")
.endObject()
.endObject()
.endObject()
.string();
client
.admin()
.indices()
.prepareCreate(index)
.addMapping(type, mapping)
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
.execute()
.actionGet();
client
.prepareIndex(index, type, "1")
.setSource(
jsonBuilder()
.startObject()
.field("field", "1", "2")
.field("included", "should be seen")
.field("excluded", "should not be seen")
.endObject())
.execute()
.actionGet();
GetResponse responseBeforeFlush =
client
.prepareGet(index, type, "1")
.setFields("_source", "included", "excluded")
.execute()
.actionGet();
client.admin().indices().prepareFlush(index).execute().actionGet();
GetResponse responseAfterFlush =
client
.prepareGet(index, type, "1")
.setFields("_source", "included", "excluded")
.execute()
.actionGet();
assertThat(responseBeforeFlush.isExists(), is(true));
assertThat(responseAfterFlush.isExists(), is(true));
assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("excluded")));
assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("field")));
assertThat(responseBeforeFlush.getSourceAsMap(), hasKey("included"));
assertThat(responseBeforeFlush.getSourceAsString(), is(responseAfterFlush.getSourceAsString()));
}
/**
* Basic test using Index & Realtime Get with internal versioning. This test ensures routing
* works correctly across versions.
*/
public void testInternalVersion() throws Exception {
createIndex("test");
final boolean routing = randomBoolean();
int numDocs = randomIntBetween(10, 20);
for (int i = 0; i < numDocs; i++) {
String routingKey = routing ? randomRealisticUnicodeOfLength(10) : null;
String id = Integer.toString(i);
assertThat(
id,
client()
.prepareIndex("test", "type1", id)
.setRouting(routingKey)
.setSource("field1", English.intToEnglish(i))
.get()
.isCreated(),
is(true));
GetResponse get =
client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(1).get();
assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
assertThat(get.getVersion(), equalTo(1L));
client()
.prepareIndex("test", "type1", id)
.setRouting(routingKey)
.setSource("field1", English.intToEnglish(i))
.execute()
.actionGet();
get = client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(2).get();
assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
assertThat(get.getVersion(), equalTo(2L));
}
assertVersionCreated(compatibilityVersion(), "test");
}
public void testIndexNameDateMathExpressions() {
DateTime now = new DateTime(DateTimeZone.UTC);
String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now);
String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1));
String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2));
createIndex(index1, index2, index3);
String dateMathExp1 = "<.marvel-{now/d}>";
String dateMathExp2 = "<.marvel-{now/d-1d}>";
String dateMathExp3 = "<.marvel-{now/d-2d}>";
client().prepareIndex(dateMathExp1, "type", "1").setSource("{}").get();
client().prepareIndex(dateMathExp2, "type", "2").setSource("{}").get();
client().prepareIndex(dateMathExp3, "type", "3").setSource("{}").get();
refresh();
SearchResponse searchResponse =
client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get();
assertHitCount(searchResponse, 3);
assertSearchHits(searchResponse, "1", "2", "3");
GetResponse getResponse = client().prepareGet(dateMathExp1, "type", "1").get();
assertThat(getResponse.isExists(), is(true));
assertThat(getResponse.getId(), equalTo("1"));
getResponse = client().prepareGet(dateMathExp2, "type", "2").get();
assertThat(getResponse.isExists(), is(true));
assertThat(getResponse.getId(), equalTo("2"));
getResponse = client().prepareGet(dateMathExp3, "type", "3").get();
assertThat(getResponse.isExists(), is(true));
assertThat(getResponse.getId(), equalTo("3"));
IndicesStatsResponse indicesStatsResponse =
client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get();
assertThat(indicesStatsResponse.getIndex(index1), notNullValue());
assertThat(indicesStatsResponse.getIndex(index2), notNullValue());
assertThat(indicesStatsResponse.getIndex(index3), notNullValue());
DeleteResponse deleteResponse = client().prepareDelete(dateMathExp1, "type", "1").get();
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
assertThat(deleteResponse.getId(), equalTo("1"));
deleteResponse = client().prepareDelete(dateMathExp2, "type", "2").get();
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
assertThat(deleteResponse.getId(), equalTo("2"));
deleteResponse = client().prepareDelete(dateMathExp3, "type", "3").get();
assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
assertThat(deleteResponse.getId(), equalTo("3"));
}
@Override
public HashMap<String, String> loadKey(String username) {
logger.info("loading password for username {}", username);
HashMap<String, String> ret = new HashMap<>();
String riverIndexName = getRiverIndexName();
refreshSearchIndex(riverIndexName);
GetResponse resp =
client.prepareGet(riverIndexName, riverName().name(), "_pwd").execute().actionGet();
if (resp.isExists()) {
if (logger.isDebugEnabled()) {
logger.debug("Password document: {}", resp.getSourceAsString());
}
Map<String, Object> newset = resp.getSource();
Set<String> keys = newset.keySet();
for (String s : keys) {
logger.info(
"Added key {} with a value of {}",
s,
XContentMapValues.nodeStringValue(newset.get(s), null));
ret.put(s, XContentMapValues.nodeStringValue(newset.get(s), null));
}
}
if (ret.isEmpty()) {
return null;
}
return ret;
}
public void testDeleteRoutingRequired()
throws ExecutionException, InterruptedException, IOException {
createIndexWithAlias();
assertAcked(
client()
.admin()
.indices()
.preparePutMapping("test")
.setType("test")
.setSource(
XContentFactory.jsonBuilder()
.startObject()
.startObject("test")
.startObject("_routing")
.field("required", true)
.endObject()
.endObject()
.endObject()));
ensureYellow("test");
int numDocs = iterations(10, 50);
IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs];
for (int i = 0; i < numDocs - 2; i++) {
indexRequestBuilders[i] =
client()
.prepareIndex("test", "test", Integer.toString(i))
.setRouting(randomAsciiOfLength(randomIntBetween(1, 10)))
.setSource("field", "value");
}
String firstDocId = Integer.toString(numDocs - 2);
indexRequestBuilders[numDocs - 2] =
client()
.prepareIndex("test", "test", firstDocId)
.setRouting("routing")
.setSource("field", "value");
String secondDocId = Integer.toString(numDocs - 1);
String secondRouting = randomAsciiOfLength(randomIntBetween(1, 10));
indexRequestBuilders[numDocs - 1] =
client()
.prepareIndex("test", "test", secondDocId)
.setRouting(secondRouting)
.setSource("field", "value");
indexRandom(true, indexRequestBuilders);
SearchResponse searchResponse = client().prepareSearch("test").get();
assertNoFailures(searchResponse);
assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs));
DeleteResponse deleteResponse =
client().prepareDelete("test", "test", firstDocId).setRouting("routing").get();
assertThat(deleteResponse.isFound(), equalTo(true));
GetResponse getResponse =
client().prepareGet("test", "test", firstDocId).setRouting("routing").get();
assertThat(getResponse.isExists(), equalTo(false));
refresh();
searchResponse = client().prepareSearch("test").get();
assertNoFailures(searchResponse);
assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1));
}
private List<KnapsackState> get(String name) throws IOException {
ImmutableList.Builder<KnapsackState> builder = ImmutableList.builder();
try {
logger.debug("get knapsack states: {}", name);
final Client client = injector.getInstance(Client.class);
createIndexIfNotExist(client);
GetResponse getResponse =
client.prepareGet(INDEX_NAME, MAPPING_NAME, name).execute().actionGet();
if (!getResponse.isExists()) {
return builder.build();
}
XContentParser parser = xContent(JSON).createParser(getResponse.getSourceAsBytes());
while (parser.nextToken() != START_ARRAY) {
// forward
}
while (parser.nextToken() != END_ARRAY) {
KnapsackState state = new KnapsackState();
builder.add(state.fromXContent(parser));
}
return builder.build();
} catch (Throwable t) {
logger.error("get settings failed", t);
return null;
}
}
@Override
public Date readDatetimeValue(String spaceKey, String propertyName) throws IOException {
Date lastDate = null;
String documentName = prepareValueStoreDocumentName(spaceKey, propertyName);
if (logger.isDebugEnabled())
logger.debug(
"Going to read datetime value from {} property for space {}. Document name is {}.",
propertyName,
spaceKey,
documentName);
refreshSearchIndex(getRiverIndexName());
GetResponse lastSeqGetResponse =
client
.prepareGet(getRiverIndexName(), riverName.name(), documentName)
.execute()
.actionGet();
if (lastSeqGetResponse.isExists()) {
Object timestamp = lastSeqGetResponse.getSourceAsMap().get(STORE_FIELD_VALUE);
if (timestamp != null) {
lastDate = DateTimeUtils.parseISODateTime(timestamp.toString());
}
} else {
if (logger.isDebugEnabled())
logger.debug("{} document doesn't exist in remore river persistent store", documentName);
}
return lastDate;
}
private void test(
String name, Class<? extends HashFunction> expectedHashFunction, boolean expectedUseType)
throws Exception {
Path zippedIndexDir = getDataPath("/org/elasticsearch/cluster/routing/" + name + ".zip");
Settings baseSettings = prepareBackwardsDataDir(zippedIndexDir);
internalCluster()
.startNode(Settings.builder().put(baseSettings).put(Node.HTTP_ENABLED, true).build());
ensureYellow("test");
GetIndexResponse getIndexResponse = client().admin().indices().prepareGetIndex().get();
assertArrayEquals(new String[] {"test"}, getIndexResponse.indices());
GetSettingsResponse getSettingsResponse =
client().admin().indices().prepareGetSettings("test").get();
assertEquals(
expectedHashFunction.getName(),
getSettingsResponse.getSetting("test", IndexMetaData.SETTING_LEGACY_ROUTING_HASH_FUNCTION));
assertEquals(
Boolean.valueOf(expectedUseType).toString(),
getSettingsResponse.getSetting("test", IndexMetaData.SETTING_LEGACY_ROUTING_USE_TYPE));
SearchResponse allDocs = client().prepareSearch("test").get();
assertSearchResponse(allDocs);
assertHitCount(allDocs, 4);
// Make sure routing works
for (SearchHit hit : allDocs.getHits().hits()) {
GetResponse get = client().prepareGet(hit.index(), hit.type(), hit.id()).get();
assertTrue(get.isExists());
}
}
public void testPrimaryRelocation() throws Exception {
Path dataPath = createTempDir();
Settings nodeSettings = nodeSettings(dataPath);
String node1 = internalCluster().startNode(nodeSettings);
String IDX = "test";
Settings idxSettings =
Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString())
.put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
.put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true)
.build();
prepareCreate(IDX).setSettings(idxSettings).addMapping("doc", "foo", "type=text").get();
client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get();
client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get();
GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get();
GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get();
assertTrue(gResp1.isExists());
assertTrue(gResp2.isExists());
assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
// Node1 has the primary, now node2 has the replica
String node2 = internalCluster().startNode(nodeSettings);
ensureGreen(IDX);
client().admin().cluster().prepareHealth().setWaitForNodes("2").get();
flushAndRefresh(IDX);
// now prevent primary from being allocated on node 1 move to node_3
String node3 = internalCluster().startNode(nodeSettings);
Settings build =
Settings.builder().put("index.routing.allocation.exclude._name", node1).build();
client().admin().indices().prepareUpdateSettings(IDX).setSettings(build).execute().actionGet();
ensureGreen(IDX);
logger.info("--> performing query");
SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
assertHitCount(resp, 2);
gResp1 = client().prepareGet(IDX, "doc", "1").get();
gResp2 = client().prepareGet(IDX, "doc", "2").get();
assertTrue(gResp1.isExists());
assertTrue(gResp2.toString(), gResp2.isExists());
assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
client().prepareIndex(IDX, "doc", "3").setSource("foo", "bar").get();
client().prepareIndex(IDX, "doc", "4").setSource("foo", "bar").get();
gResp1 = client().prepareGet(IDX, "doc", "3").setPreference("_primary").get();
gResp2 = client().prepareGet(IDX, "doc", "4").setPreference("_primary").get();
assertTrue(gResp1.isExists());
assertTrue(gResp2.isExists());
assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
}
protected ESShape getShape(Iterator<MultiGetItemResponse> iterator, int size) {
ESShapeBuilder shapeBuilder = new ESShapeBuilder(size);
for (int i = 0; i < size; i++) {
GetResponse response = iterator.next().getResponse();
if (!response.isExists()) continue;
@SuppressWarnings("unchecked")
Map<String, Object> shape = (Map<String, Object>) response.getField("shape").getValue();
@SuppressWarnings("unchecked")
List<Double> coordinates = (List<Double>) shape.get("coordinates");
shapeBuilder.addLocation(coordinates.get(1), coordinates.get(0));
}
return shapeBuilder.build();
}
public void testIndexGetAndDelete() throws ExecutionException, InterruptedException {
createIndexWithAlias();
ensureYellow("test");
int numDocs = iterations(10, 50);
for (int i = 0; i < numDocs; i++) {
IndexResponse indexResponse =
client()
.prepareIndex(indexOrAlias(), "type", Integer.toString(i))
.setSource("field", "value-" + i)
.get();
assertThat(indexResponse.isCreated(), equalTo(true));
assertThat(indexResponse.getIndex(), equalTo("test"));
assertThat(indexResponse.getType(), equalTo("type"));
assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
}
refresh();
String docId = Integer.toString(randomIntBetween(0, numDocs - 1));
GetResponse getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
assertThat(getResponse.isExists(), equalTo(true));
assertThat(getResponse.getIndex(), equalTo("test"));
assertThat(getResponse.getType(), equalTo("type"));
assertThat(getResponse.getId(), equalTo(docId));
DeleteResponse deleteResponse = client().prepareDelete(indexOrAlias(), "type", docId).get();
assertThat(deleteResponse.isFound(), equalTo(true));
assertThat(deleteResponse.getIndex(), equalTo("test"));
assertThat(deleteResponse.getType(), equalTo("type"));
assertThat(deleteResponse.getId(), equalTo(docId));
getResponse = client().prepareGet(indexOrAlias(), "type", docId).get();
assertThat(getResponse.isExists(), equalTo(false));
refresh();
SearchResponse searchResponse = client().prepareSearch(indexOrAlias()).get();
assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs - 1));
}
public void testReplicaToPrimaryPromotion() throws Exception {
Path dataPath = createTempDir();
Settings nodeSettings = nodeSettings(dataPath);
String node1 = internalCluster().startNode(nodeSettings);
String IDX = "test";
Settings idxSettings =
Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
.put(IndexMetaData.SETTING_DATA_PATH, dataPath.toAbsolutePath().toString())
.put(IndexMetaData.SETTING_SHADOW_REPLICAS, true)
.put(IndexMetaData.SETTING_SHARED_FILESYSTEM, true)
.build();
prepareCreate(IDX).setSettings(idxSettings).addMapping("doc", "foo", "type=text").get();
client().prepareIndex(IDX, "doc", "1").setSource("foo", "bar").get();
client().prepareIndex(IDX, "doc", "2").setSource("foo", "bar").get();
GetResponse gResp1 = client().prepareGet(IDX, "doc", "1").get();
GetResponse gResp2 = client().prepareGet(IDX, "doc", "2").get();
assertTrue(gResp1.isExists());
assertTrue(gResp2.isExists());
assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
// Node1 has the primary, now node2 has the replica
String node2 = internalCluster().startNode(nodeSettings);
ensureGreen(IDX);
client().admin().cluster().prepareHealth().setWaitForNodes("2").get();
flushAndRefresh(IDX);
logger.info("--> stopping node1 [{}]", node1);
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node1));
ensureYellow(IDX);
logger.info("--> performing query");
SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).get();
assertHitCount(resp, 2);
gResp1 = client().prepareGet(IDX, "doc", "1").get();
gResp2 = client().prepareGet(IDX, "doc", "2").get();
assertTrue(gResp1.isExists());
assertTrue(gResp2.toString(), gResp2.isExists());
assertThat(gResp1.getSource().get("foo"), equalTo("bar"));
assertThat(gResp2.getSource().get("foo"), equalTo("bar"));
client().prepareIndex(IDX, "doc", "1").setSource("foo", "foobar").get();
client().prepareIndex(IDX, "doc", "2").setSource("foo", "foobar").get();
gResp1 = client().prepareGet(IDX, "doc", "1").get();
gResp2 = client().prepareGet(IDX, "doc", "2").get();
assertTrue(gResp1.isExists());
assertTrue(gResp2.toString(), gResp2.isExists());
assertThat(gResp1.getSource().get("foo"), equalTo("foobar"));
assertThat(gResp2.getSource().get("foo"), equalTo("foobar"));
}
/**
* Basic test using Index & Realtime Get with external versioning. This test ensures routing
* works correctly across versions.
*/
public void testExternalVersion() throws Exception {
createIndex("test");
final boolean routing = randomBoolean();
int numDocs = randomIntBetween(10, 20);
for (int i = 0; i < numDocs; i++) {
String id = Integer.toString(i);
String routingKey = routing ? randomRealisticUnicodeOfLength(10) : null;
final long version = randomIntBetween(0, Integer.MAX_VALUE);
client()
.prepareIndex("test", "type1", id)
.setRouting(routingKey)
.setVersion(version)
.setVersionType(VersionType.EXTERNAL)
.setSource("field1", English.intToEnglish(i))
.get();
GetResponse get =
client().prepareGet("test", "type1", id).setRouting(routingKey).setVersion(version).get();
assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
assertThat(get.getVersion(), equalTo(version));
final long nextVersion = version + randomIntBetween(0, Integer.MAX_VALUE);
client()
.prepareIndex("test", "type1", id)
.setRouting(routingKey)
.setVersion(nextVersion)
.setVersionType(VersionType.EXTERNAL)
.setSource("field1", English.intToEnglish(i))
.get();
get =
client()
.prepareGet("test", "type1", id)
.setRouting(routingKey)
.setVersion(nextVersion)
.get();
assertThat("Document with ID " + id + " should exist but doesn't", get.isExists(), is(true));
assertThat(get.getVersion(), equalTo(nextVersion));
}
}
@SuppressWarnings("unchecked")
protected <T extends ESEntity> T buildEntityFromGetResponse(
Class<T> entityClass, MultiGetItemResponse item) {
GetResponse response = item.getResponse();
if (!response.isExists())
throw new DaoException(
String.format(
"Entity %s does not exist in %s/%s",
response.getId(), response.getIndex(), response.getType()));
if (entityClass == null) throw new IllegalArgumentException("Provided Entity class is null");
else if (entityClass.equals(ESNode.class))
return (T) ESNode.Builder.buildFromGetReponse(response);
else if (entityClass.equals(ESWay.class))
return (T) ESWay.Builder.buildFromGetReponse(response);
else throw new IllegalArgumentException(entityClass.getSimpleName() + " is not a known Entity");
}
protected void createRiver(String jsonDefinition, String river, Object... args) throws Exception {
logger.info("Create river [{}]", river);
String setting = getJsonSettings(jsonDefinition, args);
logger.info("River setting [{}]", setting);
node.client().prepareIndex("_river", river, "_meta").setSource(setting).execute().actionGet();
logger.debug("Running Cluster Health");
ClusterHealthResponse clusterHealth =
node.client()
.admin()
.cluster()
.health(clusterHealthRequest().waitForGreenStatus())
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
GetResponse response =
getNode().client().prepareGet("_river", river, "_meta").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
}
public RiverState load(Client client) throws IOException {
if (coordinateIndex == null || coordinateType == null || coordinateId == null) {
return this;
}
GetResponse get = null;
try {
get = client.prepareGet(coordinateIndex, coordinateType, coordinateId).execute().actionGet();
} catch (Exception e) {
// ignore
}
if (get != null && get.isExists()) {
XContentParser parser =
XContentFactory.xContent(XContentType.JSON).createParser(get.getSourceAsBytes());
fromXContent(parser);
} else {
counter = 0L;
}
return this;
}
/** Reconfigure the river. Must be stopped! */
public synchronized void reconfigure() {
if (!closed) throw new IllegalStateException("Remote River must be stopped to reconfigure it!");
logger.info("reconfiguring Remote River");
String riverIndexName = getRiverIndexName();
refreshSearchIndex(riverIndexName);
GetResponse resp =
client.prepareGet(riverIndexName, riverName().name(), "_meta").execute().actionGet();
if (resp.isExists()) {
if (logger.isDebugEnabled()) {
logger.debug("Configuration document: {}", resp.getSourceAsString());
}
Map<String, Object> newset = resp.getSource();
configure(newset);
} else {
throw new IllegalStateException(
"Configuration document not found to reconfigure remote river " + riverName().name());
}
}
@CheckForNull
@Override
public DOMAIN getNullableByKey(KEY key) {
GetRequestBuilder request =
client
.prepareGet()
.setType(this.getIndexType())
.setIndex(this.getIndexName())
.setId(this.getKeyValue(key))
.setFetchSource(true)
.setRouting(this.getKeyValue(key));
GetResponse response = request.get();
if (response.isExists()) {
return toDoc(response.getSource());
}
return null;
}
@Override
public Article get(long id) {
GetResponse getResponse =
transportClient.prepareGet(INDICE, TYPE, String.valueOf(id)).execute().actionGet();
if (getResponse.isExists()) {
Map<String, Object> map = getResponse.getSource();
String title = (String) map.get(TITLE);
String summary = (String) map.get(SUMMARY);
String idStr = getResponse.getId();
logger.debug("get id : {}", idStr);
Article article = new Article();
article.setId(Long.parseLong(idStr));
article.setTitle(title);
article.setSummary(summary);
return article;
} else {
return null;
}
}
@Override
public Table load(String key) {
logger.info("Load called for: " + key);
GetResponse response =
elasticsearchConnection
.getClient()
.prepareGet()
.setIndex(TABLE_META_INDEX)
.setType(TABLE_META_TYPE)
.setId(key)
.execute()
.actionGet();
if (!response.isExists()) {
return null;
}
try {
return objectMapper.readValue(response.getSourceAsBytes(), Table.class);
} catch (Exception e) {
throw new RuntimeException("Error getting data for table: " + key);
}
}
@Test
public void realtimeGetWithCompress() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
client
.admin()
.indices()
.prepareCreate("test")
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
.addMapping(
"type",
jsonBuilder()
.startObject()
.startObject("type")
.startObject("_source")
.field("compress", true)
.endObject()
.endObject()
.endObject())
.execute()
.actionGet();
ClusterHealthResponse clusterHealth =
client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
StringBuilder sb = new StringBuilder();
for (int i = 0; i < 10000; i++) {
sb.append((char) i);
}
String fieldValue = sb.toString();
client.prepareIndex("test", "type", "1").setSource("field", fieldValue).execute().actionGet();
// realtime get
GetResponse getResponse = client.prepareGet("test", "type", "1").execute().actionGet();
assertThat(getResponse.isExists(), equalTo(true));
assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo(fieldValue));
}
/**
* Get the river definition by its name
*
* @param name
* @return
*/
public T get(String name) {
if (logger.isDebugEnabled()) logger.debug("get({})", name);
T river = null;
if (name != null) {
GetRequestBuilder rb = new GetRequestBuilder(client, SMDSearchProperties.ES_META_INDEX);
rb.setType(SMDSearchProperties.ES_META_RIVERS);
rb.setId(name);
try {
GetResponse response = rb.execute().actionGet();
if (response.isExists()) {
river = getHelper().toRiver(buildInstance(), response.getSourceAsMap());
}
} catch (IndexMissingException e) {
// Index does not exists, so RIVER does not exist...
}
}
if (logger.isDebugEnabled()) logger.debug("/get({})={}", name, river);
return river;
}
public final Map<String, GetField> getModelFields(
final ModelIdT id, final Logger logger, final Marker logMarker, final String... fieldNames)
throws InvalidModelException, IoExceptionT, NoSuchModelExceptionT {
logger.debug(logMarker, "getting model {} from index {}", id, indexName);
try {
final GetResponse response =
client
.prepareGet(indexName, documentType, id.toString())
.setFields(fieldNames)
.execute()
.actionGet();
if (!response.isExists()) {
throw exceptionFactory.newNoSuchModelException(id);
}
return response.getFields();
} catch (final IndexNotFoundException e) {
logger.warn(logMarker, "tried to get model {} from missing index {}", id, indexName);
throw exceptionFactory.newNoSuchModelException(id);
} catch (final ElasticsearchException e) {
throw exceptionFactory.newIoException(
e, String.format("error getting model %s from index %s", id, indexName));
}
}
@Before
public void setupElasticAndRedis() throws Exception {
Settings globalSettings = settingsBuilder().loadFromClasspath("settings.yml").build();
String json = copyToStringFromClasspath("/simple-redis-river.json");
Settings riverSettings = settingsBuilder().loadFromSource(json).build();
index = riverSettings.get("index.name");
channel = riverSettings.get("redis.channels").split(",")[0];
String hostname = riverSettings.get("redis.hostname");
int port = riverSettings.getAsInt("redis.port", 6379);
logger.debug("Connecting to Redis [hostname={} port={}]...", hostname, port);
jedis = new Jedis(hostname, port, 0);
logger.debug("... connected");
logger.debug("Starting local elastic...");
node = nodeBuilder().local(true).settings(globalSettings).node();
logger.info("Create river [{}]", river);
node.client().prepareIndex("_river", river, "_meta").setSource(json).execute().actionGet();
logger.debug("Running Cluster Health");
ClusterHealthResponse clusterHealth =
node.client()
.admin()
.cluster()
.health(clusterHealthRequest().waitForGreenStatus())
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
GetResponse response = node.client().prepareGet("_river", river, "_meta").execute().actionGet();
assertTrue(response.isExists());
logger.debug("...elasticized ok");
}
public final ModelT getModelById(
final ModelIdT id,
final Logger logger,
final Marker logMarker,
final ModelFactory<ModelEntryT> modelFactory)
throws InvalidModelException, IoExceptionT, NoSuchModelExceptionT {
logger.debug(logMarker, "getting model {} from index {}", id, indexName);
try {
final GetResponse response =
client.prepareGet(indexName, documentType, id.toString()).execute().actionGet();
if (!response.isExists()) {
throw exceptionFactory.newNoSuchModelException(id);
}
return modelFactory
.createModelEntry(response.getId(), response.getSourceAsBytesRef())
.getModel();
} catch (final IndexNotFoundException e) {
logger.warn(logMarker, "tried to get model {} from missing index {}", id, indexName);
throw exceptionFactory.newNoSuchModelException(id);
} catch (final ElasticsearchException e) {
throw exceptionFactory.newIoException(
e, String.format("error getting model %s from index %s", id, indexName));
}
}
@Test
public void simpleGetTests() {
client.admin().indices().prepareDelete().execute().actionGet();
client
.admin()
.indices()
.prepareCreate("test")
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
.execute()
.actionGet();
ClusterHealthResponse clusterHealth =
client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
GetResponse response = client.prepareGet("test", "type1", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(false));
logger.info("--> index doc 1");
client
.prepareIndex("test", "type1", "1")
.setSource("field1", "value1", "field2", "value2")
.execute()
.actionGet();
logger.info("--> realtime get 1");
response = client.prepareGet("test", "type1", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1"));
assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));
logger.info("--> realtime get 1 (no source)");
response =
client
.prepareGet("test", "type1", "1")
.setFields(Strings.EMPTY_ARRAY)
.execute()
.actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getSourceAsBytes(), nullValue());
logger.info("--> realtime get 1 (no type)");
response = client.prepareGet("test", null, "1").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1"));
assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));
logger.info("--> non realtime get 1");
response = client.prepareGet("test", "type1", "1").setRealtime(false).execute().actionGet();
assertThat(response.isExists(), equalTo(false));
logger.info("--> realtime fetch of field (requires fetching parsing source)");
response = client.prepareGet("test", "type1", "1").setFields("field1").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getSourceAsBytes(), nullValue());
assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1"));
assertThat(response.getField("field2"), nullValue());
logger.info("--> flush the index, so we load it from it");
client.admin().indices().prepareFlush().execute().actionGet();
logger.info("--> realtime get 1 (loaded from index)");
response = client.prepareGet("test", "type1", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1"));
assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));
logger.info("--> non realtime get 1 (loaded from index)");
response = client.prepareGet("test", "type1", "1").setRealtime(false).execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1"));
assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));
logger.info("--> realtime fetch of field (loaded from index)");
response = client.prepareGet("test", "type1", "1").setFields("field1").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getSourceAsBytes(), nullValue());
assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1"));
assertThat(response.getField("field2"), nullValue());
logger.info("--> update doc 1");
client
.prepareIndex("test", "type1", "1")
.setSource("field1", "value1_1", "field2", "value2_1")
.execute()
.actionGet();
logger.info("--> realtime get 1");
response = client.prepareGet("test", "type1", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1_1"));
assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2_1"));
logger.info("--> update doc 1 again");
client
.prepareIndex("test", "type1", "1")
.setSource("field1", "value1_2", "field2", "value2_2")
.execute()
.actionGet();
response = client.prepareGet("test", "type1", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1_2"));
assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2_2"));
DeleteResponse deleteResponse =
client.prepareDelete("test", "type1", "1").execute().actionGet();
assertThat(deleteResponse.isNotFound(), equalTo(false));
response = client.prepareGet("test", "type1", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(false));
}
@Test
public void testSimpleTTL() throws Exception {
assertAcked(
prepareCreate("test")
.addMapping(
"type1",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.endObject()
.endObject()
.endObject())
.addMapping(
"type2",
XContentFactory.jsonBuilder()
.startObject()
.startObject("type2")
.startObject("_timestamp")
.field("enabled", true)
.field("store", "yes")
.endObject()
.startObject("_ttl")
.field("enabled", true)
.field("default", "1d")
.endObject()
.endObject()
.endObject()));
ensureYellow("test");
final NumShards test = getNumShards("test");
long providedTTLValue = 3000;
logger.info("--> checking ttl");
// Index one doc without routing, one doc with routing, one doc with not TTL and no default and
// one doc with default TTL
long now = System.currentTimeMillis();
IndexResponse indexResponse =
client()
.prepareIndex("test", "type1", "1")
.setSource("field1", "value1")
.setTimestamp(String.valueOf(now))
.setTTL(providedTTLValue)
.setRefresh(true)
.get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client()
.prepareIndex("test", "type1", "with_routing")
.setSource("field1", "value1")
.setTimestamp(String.valueOf(now))
.setTTL(providedTTLValue)
.setRouting("routing")
.setRefresh(true)
.get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client().prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").get();
assertThat(indexResponse.isCreated(), is(true));
indexResponse =
client().prepareIndex("test", "type2", "default_ttl").setSource("field1", "value1").get();
assertThat(indexResponse.isCreated(), is(true));
// realtime get check
long currentTime = System.currentTimeMillis();
GetResponse getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
long ttl0;
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// verify the ttl is still decreasing when going to the replica
currentTime = System.currentTimeMillis();
getResponse = client().prepareGet("test", "type1", "1").setFields("_ttl").get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// non realtime get (stored)
currentTime = System.currentTimeMillis();
getResponse =
client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// non realtime get going the replica
currentTime = System.currentTimeMillis();
getResponse =
client().prepareGet("test", "type1", "1").setFields("_ttl").setRealtime(false).get();
if (getResponse.isExists()) {
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, lessThanOrEqualTo(providedTTLValue - (currentTime - now)));
} else {
assertThat(providedTTLValue - (currentTime - now), lessThanOrEqualTo(0l));
}
// no TTL provided so no TTL fetched
getResponse =
client()
.prepareGet("test", "type1", "no_ttl")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.getField("_ttl"), nullValue());
// no TTL provided make sure it has default TTL
getResponse =
client()
.prepareGet("test", "type2", "default_ttl")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
ttl0 = ((Number) getResponse.getField("_ttl").getValue()).longValue();
assertThat(ttl0, greaterThan(0L));
IndicesStatsResponse response =
client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
assertThat(
response.getIndices().get("test").getTotal().getIndexing().getTotal().getDeleteCount(),
equalTo(0L));
// make sure the purger has done its job for all indexed docs that are expired
long shouldBeExpiredDate = now + providedTTLValue + PURGE_INTERVAL + 2000;
currentTime = System.currentTimeMillis();
if (shouldBeExpiredDate - currentTime > 0) {
Thread.sleep(shouldBeExpiredDate - currentTime);
}
// We can't assume that after waiting for ttl + purgeInterval (waitTime) that the document have
// actually been deleted.
// The ttl purging happens in the background in a different thread, and might not have been
// completed after waiting for waitTime.
// But we can use index statistics' delete count to be sure that deletes have been executed,
// that must be incremented before
// ttl purging has finished.
logger.info("--> checking purger");
assertThat(
awaitBusy(
new Predicate<Object>() {
@Override
public boolean apply(Object input) {
if (rarely()) {
client().admin().indices().prepareFlush("test").get();
} else if (rarely()) {
client().admin().indices().prepareOptimize("test").setMaxNumSegments(1).get();
}
IndicesStatsResponse response =
client().admin().indices().prepareStats("test").clear().setIndexing(true).get();
// TTL deletes two docs, but it is indexed in the primary shard and replica shard.
return response
.getIndices()
.get("test")
.getTotal()
.getIndexing()
.getTotal()
.getDeleteCount()
== 2L * test.dataCopies;
}
},
5,
TimeUnit.SECONDS),
equalTo(true));
// realtime get check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// replica realtime get check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(true)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// Need to run a refresh, in order for the non realtime get to work.
client().admin().indices().prepareRefresh("test").execute().actionGet();
// non realtime get (stored) check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
// non realtime get going the replica check
getResponse =
client()
.prepareGet("test", "type1", "1")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
getResponse =
client()
.prepareGet("test", "type1", "with_routing")
.setRouting("routing")
.setFields("_ttl")
.setRealtime(false)
.execute()
.actionGet();
assertThat(getResponse.isExists(), equalTo(false));
}
@Test
public void testGetDocWithMultivaluedFields() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
// fine
}
String mapping1 =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("field")
.field("type", "string")
.field("store", "yes")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
String mapping2 =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type2")
.startObject("properties")
.startObject("field")
.field("type", "string")
.field("store", "yes")
.endObject()
.endObject()
.startObject("_source")
.field("enabled", false)
.endObject()
.endObject()
.endObject()
.string();
client
.admin()
.indices()
.prepareCreate("test")
.addMapping("type1", mapping1)
.addMapping("type2", mapping2)
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
.execute()
.actionGet();
ClusterHealthResponse clusterHealth =
client.admin().cluster().health(clusterHealthRequest().waitForGreenStatus()).actionGet();
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
GetResponse response = client.prepareGet("test", "type1", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(false));
response = client.prepareGet("test", "type2", "1").execute().actionGet();
assertThat(response.isExists(), equalTo(false));
client
.prepareIndex("test", "type1", "1")
.setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
.execute()
.actionGet();
client
.prepareIndex("test", "type2", "1")
.setSource(jsonBuilder().startObject().field("field", "1", "2").endObject())
.execute()
.actionGet();
response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getId(), equalTo("1"));
assertThat(response.getType(), equalTo("type1"));
assertThat(response.getFields().size(), equalTo(1));
assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
equalTo("1"));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
equalTo("2"));
response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getType(), equalTo("type2"));
assertThat(response.getId(), equalTo("1"));
assertThat(response.getFields().size(), equalTo(1));
assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
equalTo("1"));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
equalTo("2"));
// Now test values being fetched from stored fields.
client.admin().indices().prepareRefresh("test").execute().actionGet();
response = client.prepareGet("test", "type1", "1").setFields("field").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getId(), equalTo("1"));
assertThat(response.getFields().size(), equalTo(1));
assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
equalTo("1"));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
equalTo("2"));
response = client.prepareGet("test", "type2", "1").setFields("field").execute().actionGet();
assertThat(response.isExists(), equalTo(true));
assertThat(response.getId(), equalTo("1"));
assertThat(response.getFields().size(), equalTo(1));
assertThat(response.getFields().get("field").getValues().size(), equalTo(1));
assertThat(((List) response.getFields().get("field").getValues().get(0)).size(), equalTo(2));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(0).toString(),
equalTo("1"));
assertThat(
((List) response.getFields().get("field").getValues().get(0)).get(1).toString(),
equalTo("2"));
}
@Test
@Slow
public void testSnapshotOperations() throws Exception {
startNode("server1", getClassDefaultSettings());
// get the environment, so we can clear the work dir when needed
Environment environment =
((InternalNode) node("server1")).injector().getInstance(Environment.class);
logger.info("Running Cluster Health (waiting for node to startup properly)");
ClusterHealthResponse clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForGreenStatus())
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
// Translog tests
logger.info("Creating index [{}]", "test");
client("server1").admin().indices().prepareCreate("test").execute().actionGet();
// create a mapping
PutMappingResponse putMappingResponse =
client("server1")
.admin()
.indices()
.preparePutMapping("test")
.setType("type1")
.setSource(mappingSource())
.execute()
.actionGet();
assertThat(putMappingResponse.isAcknowledged(), equalTo(true));
// verify that mapping is there
ClusterStateResponse clusterState =
client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue());
// create two and delete the first
logger.info("Indexing #1");
client("server1")
.index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")))
.actionGet();
logger.info("Indexing #2");
client("server1")
.index(Requests.indexRequest("test").type("type1").id("2").source(source("2", "test")))
.actionGet();
// perform snapshot to the index
logger.info("Gateway Snapshot");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Deleting #1");
client("server1").delete(deleteRequest("test").type("type1").id("1")).actionGet();
// perform snapshot to the index
logger.info("Gateway Snapshot");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Gateway Snapshot (should be a no op)");
// do it again, it should be a no op
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Closing the server");
closeNode("server1");
logger.info(
"Starting the server, should recover from the gateway (only translog should be populated)");
startNode("server1");
logger.info("Running Cluster Health (wait for the shards to startup)");
clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
// verify that mapping is there
clusterState = client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue());
logger.info("Getting #1, should not exists");
GetResponse getResponse =
client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
assertThat(getResponse.isExists(), equalTo(false));
logger.info("Getting #2");
getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
// Now flush and add some data (so we have index recovery as well)
logger.info(
"Flushing, so we have actual content in the index files (#2 should be in the index)");
client("server1").admin().indices().flush(flushRequest("test")).actionGet();
logger.info("Indexing #3, so we have something in the translog as well");
client("server1")
.index(Requests.indexRequest("test").type("type1").id("3").source(source("3", "test")))
.actionGet();
logger.info("Gateway Snapshot");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Gateway Snapshot (should be a no op)");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Closing the server");
closeNode("server1");
logger.info(
"Starting the server, should recover from the gateway (both index and translog) and reuse work dir");
startNode("server1");
logger.info("Running Cluster Health (wait for the shards to startup)");
clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
logger.info("Getting #1, should not exists");
getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
assertThat(getResponse.isExists(), equalTo(false));
logger.info("Getting #2 (not from the translog, but from the index)");
getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
logger.info("Getting #3 (from the translog)");
getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));
logger.info("Closing the server");
closeNode("server1");
logger.info("Clearing cluster data dir, so there will be a full recovery from the gateway");
FileSystemUtils.deleteRecursively(environment.dataWithClusterFiles());
logger.info(
"Starting the server, should recover from the gateway (both index and translog) without reusing work dir");
startNode("server1");
logger.info("Running Cluster Health (wait for the shards to startup)");
clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
logger.info("Getting #1, should not exists");
getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
assertThat(getResponse.isExists(), equalTo(false));
logger.info("Getting #2 (not from the translog, but from the index)");
getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
logger.info("Getting #3 (from the translog)");
getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));
logger.info(
"Flushing, so we have actual content in the index files (#3 should be in the index now as well)");
client("server1").admin().indices().flush(flushRequest("test")).actionGet();
logger.info("Gateway Snapshot");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Gateway Snapshot (should be a no op)");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Closing the server");
closeNode("server1");
logger.info(
"Starting the server, should recover from the gateway (just from the index, nothing in the translog)");
startNode("server1");
logger.info("Running Cluster Health (wait for the shards to startup)");
clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
logger.info("Getting #1, should not exists");
getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
assertThat(getResponse.isExists(), equalTo(false));
logger.info("Getting #2 (not from the translog, but from the index)");
getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
logger.info("Getting #3 (not from the translog, but from the index)");
getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));
logger.info("Deleting the index");
client("server1").admin().indices().delete(deleteIndexRequest("test")).actionGet();
}
