@Test
public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throws Exception {
client.admin().indices().prepareDelete().execute().actionGet();
String index = "test";
String type = "type1";
String mapping =
jsonBuilder()
.startObject()
.startObject("source_excludes")
.startObject("_source")
.array("includes", "included")
.array("exlcudes", "excluded")
.endObject()
.endObject()
.endObject()
.string();
client
.admin()
.indices()
.prepareCreate(index)
.addMapping(type, mapping)
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1))
.execute()
.actionGet();
client
.prepareIndex(index, type, "1")
.setSource(
jsonBuilder()
.startObject()
.field("field", "1", "2")
.field("included", "should be seen")
.field("excluded", "should not be seen")
.endObject())
.execute()
.actionGet();
GetResponse responseBeforeFlush =
client
.prepareGet(index, type, "1")
.setFields("_source", "included", "excluded")
.execute()
.actionGet();
client.admin().indices().prepareFlush(index).execute().actionGet();
GetResponse responseAfterFlush =
client
.prepareGet(index, type, "1")
.setFields("_source", "included", "excluded")
.execute()
.actionGet();
assertThat(responseBeforeFlush.isExists(), is(true));
assertThat(responseAfterFlush.isExists(), is(true));
assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("excluded")));
assertThat(responseBeforeFlush.getSourceAsMap(), not(hasKey("field")));
assertThat(responseBeforeFlush.getSourceAsMap(), hasKey("included"));
assertThat(responseBeforeFlush.getSourceAsString(), is(responseAfterFlush.getSourceAsString()));
}
@Override
public HashMap<String, String> loadKey(String username) {
logger.info("loading password for username {}", username);
HashMap<String, String> ret = new HashMap<>();
String riverIndexName = getRiverIndexName();
refreshSearchIndex(riverIndexName);
GetResponse resp =
client.prepareGet(riverIndexName, riverName().name(), "_pwd").execute().actionGet();
if (resp.isExists()) {
if (logger.isDebugEnabled()) {
logger.debug("Password document: {}", resp.getSourceAsString());
}
Map<String, Object> newset = resp.getSource();
Set<String> keys = newset.keySet();
for (String s : keys) {
logger.info(
"Added key {} with a value of {}",
s,
XContentMapValues.nodeStringValue(newset.get(s), null));
ret.put(s, XContentMapValues.nodeStringValue(newset.get(s), null));
}
}
if (ret.isEmpty()) {
return null;
}
return ret;
}
@Override
public <T> T mapResult(GetResponse response, Class<T> clazz) {
T result = mapEntity(response.getSourceAsString(), clazz);
if (result != null) {
setPersistentEntityId(result, response.getId(), clazz);
}
return result;
}
public T get(String id) {
GetResponse result = client.prepareGet(index, entity, id).execute().actionGet();
String source = result.getSourceAsString();
System.out.println("Loaded SLA: " + source);
try {
T entity = mapper.readValue(source, clazz);
entity.setId(result.getId());
logger.info("Successfully obtained entity with id [{}]", id);
return entity;
} catch (IOException e) {
logger.error("Error retrieving entity with id " + id, e);
return null;
}
}
@Override
public final String esGet(String index, String type, String id) {
try {
final GetResponse gr = client.prepareGet(index, type, id).execute().actionGet();
if (!gr.exists()) return "Doesn't exist";
return gr.getSourceAsString();
} catch (ElasticSearchException e) {
log.debug("ElasticSearchException {}", e);
return e.getMessage();
}
}
/** Reconfigure the river. Must be stopped! */
public synchronized void reconfigure() {
if (!closed) throw new IllegalStateException("Remote River must be stopped to reconfigure it!");
logger.info("reconfiguring Remote River");
String riverIndexName = getRiverIndexName();
refreshSearchIndex(riverIndexName);
GetResponse resp =
client.prepareGet(riverIndexName, riverName().name(), "_meta").execute().actionGet();
if (resp.isExists()) {
if (logger.isDebugEnabled()) {
logger.debug("Configuration document: {}", resp.getSourceAsString());
}
Map<String, Object> newset = resp.getSource();
configure(newset);
} else {
throw new IllegalStateException(
"Configuration document not found to reconfigure remote river " + riverName().name());
}
}
@Test
@Slow
public void testSnapshotOperations() throws Exception {
startNode("server1", getClassDefaultSettings());
// get the environment, so we can clear the work dir when needed
Environment environment =
((InternalNode) node("server1")).injector().getInstance(Environment.class);
logger.info("Running Cluster Health (waiting for node to startup properly)");
ClusterHealthResponse clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForGreenStatus())
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.GREEN));
// Translog tests
logger.info("Creating index [{}]", "test");
client("server1").admin().indices().prepareCreate("test").execute().actionGet();
// create a mapping
PutMappingResponse putMappingResponse =
client("server1")
.admin()
.indices()
.preparePutMapping("test")
.setType("type1")
.setSource(mappingSource())
.execute()
.actionGet();
assertThat(putMappingResponse.isAcknowledged(), equalTo(true));
// verify that mapping is there
ClusterStateResponse clusterState =
client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue());
// create two and delete the first
logger.info("Indexing #1");
client("server1")
.index(Requests.indexRequest("test").type("type1").id("1").source(source("1", "test")))
.actionGet();
logger.info("Indexing #2");
client("server1")
.index(Requests.indexRequest("test").type("type1").id("2").source(source("2", "test")))
.actionGet();
// perform snapshot to the index
logger.info("Gateway Snapshot");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Deleting #1");
client("server1").delete(deleteRequest("test").type("type1").id("1")).actionGet();
// perform snapshot to the index
logger.info("Gateway Snapshot");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Gateway Snapshot (should be a no op)");
// do it again, it should be a no op
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Closing the server");
closeNode("server1");
logger.info(
"Starting the server, should recover from the gateway (only translog should be populated)");
startNode("server1");
logger.info("Running Cluster Health (wait for the shards to startup)");
clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
// verify that mapping is there
clusterState = client("server1").admin().cluster().state(clusterStateRequest()).actionGet();
assertThat(clusterState.getState().metaData().index("test").mapping("type1"), notNullValue());
logger.info("Getting #1, should not exists");
GetResponse getResponse =
client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
assertThat(getResponse.isExists(), equalTo(false));
logger.info("Getting #2");
getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
// Now flush and add some data (so we have index recovery as well)
logger.info(
"Flushing, so we have actual content in the index files (#2 should be in the index)");
client("server1").admin().indices().flush(flushRequest("test")).actionGet();
logger.info("Indexing #3, so we have something in the translog as well");
client("server1")
.index(Requests.indexRequest("test").type("type1").id("3").source(source("3", "test")))
.actionGet();
logger.info("Gateway Snapshot");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Gateway Snapshot (should be a no op)");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Closing the server");
closeNode("server1");
logger.info(
"Starting the server, should recover from the gateway (both index and translog) and reuse work dir");
startNode("server1");
logger.info("Running Cluster Health (wait for the shards to startup)");
clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
logger.info("Getting #1, should not exists");
getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
assertThat(getResponse.isExists(), equalTo(false));
logger.info("Getting #2 (not from the translog, but from the index)");
getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
logger.info("Getting #3 (from the translog)");
getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));
logger.info("Closing the server");
closeNode("server1");
logger.info("Clearing cluster data dir, so there will be a full recovery from the gateway");
FileSystemUtils.deleteRecursively(environment.dataWithClusterFiles());
logger.info(
"Starting the server, should recover from the gateway (both index and translog) without reusing work dir");
startNode("server1");
logger.info("Running Cluster Health (wait for the shards to startup)");
clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
logger.info("Getting #1, should not exists");
getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
assertThat(getResponse.isExists(), equalTo(false));
logger.info("Getting #2 (not from the translog, but from the index)");
getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
logger.info("Getting #3 (from the translog)");
getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));
logger.info(
"Flushing, so we have actual content in the index files (#3 should be in the index now as well)");
client("server1").admin().indices().flush(flushRequest("test")).actionGet();
logger.info("Gateway Snapshot");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Gateway Snapshot (should be a no op)");
client("server1").admin().indices().gatewaySnapshot(gatewaySnapshotRequest("test")).actionGet();
logger.info("Closing the server");
closeNode("server1");
logger.info(
"Starting the server, should recover from the gateway (just from the index, nothing in the translog)");
startNode("server1");
logger.info("Running Cluster Health (wait for the shards to startup)");
clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus().waitForActiveShards(1))
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(clusterHealth.getStatus(), equalTo(ClusterHealthStatus.YELLOW));
logger.info("Getting #1, should not exists");
getResponse = client("server1").get(getRequest("test").type("type1").id("1")).actionGet();
assertThat(getResponse.isExists(), equalTo(false));
logger.info("Getting #2 (not from the translog, but from the index)");
getResponse = client("server1").get(getRequest("test").type("type1").id("2")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("2", "test")));
logger.info("Getting #3 (not from the translog, but from the index)");
getResponse = client("server1").get(getRequest("test").type("type1").id("3")).actionGet();
assertThat(getResponse.getSourceAsString(), equalTo(source("3", "test")));
logger.info("Deleting the index");
client("server1").admin().indices().delete(deleteIndexRequest("test")).actionGet();
}
@Test
@TestLogging(value = "cluster.service:TRACE,action.get:TRACE")
public void testSimpleRecovery() throws Exception {
assertAcked(prepareCreate("test", 1).execute().actionGet(5000));
NumShards numShards = getNumShards("test");
logger.info("Running Cluster Health");
ensureYellow();
client()
.index(indexRequest("test").type("type1").id("1").source(source("1", "test")))
.actionGet();
FlushResponse flushResponse =
client().admin().indices().flush(flushRequest("test")).actionGet();
assertThat(flushResponse.getTotalShards(), equalTo(numShards.totalNumShards));
assertThat(flushResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
assertThat(flushResponse.getFailedShards(), equalTo(0));
client()
.index(indexRequest("test").type("type1").id("2").source(source("2", "test")))
.actionGet();
RefreshResponse refreshResponse =
client().admin().indices().refresh(refreshRequest("test")).actionGet();
assertThat(refreshResponse.getTotalShards(), equalTo(numShards.totalNumShards));
assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.numPrimaries));
assertThat(refreshResponse.getFailedShards(), equalTo(0));
allowNodes("test", 2);
logger.info("Running Cluster Health");
ensureGreen();
GetResponse getResult;
for (int i = 0; i < 5; i++) {
getResult =
client()
.get(getRequest("test").type("type1").id("1").operationThreaded(false))
.actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
getResult =
client()
.get(getRequest("test").type("type1").id("1").operationThreaded(false))
.actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
getResult =
client()
.get(getRequest("test").type("type1").id("2").operationThreaded(true))
.actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
getResult =
client()
.get(getRequest("test").type("type1").id("2").operationThreaded(true))
.actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
}
// now start another one so we move some primaries
allowNodes("test", 3);
Thread.sleep(200);
logger.info("Running Cluster Health");
ensureGreen();
for (int i = 0; i < 5; i++) {
getResult = client().get(getRequest("test").type("type1").id("1")).actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
getResult = client().get(getRequest("test").type("type1").id("1")).actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
getResult = client().get(getRequest("test").type("type1").id("1")).actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("1", "test")));
getResult =
client()
.get(getRequest("test").type("type1").id("2").operationThreaded(true))
.actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
getResult =
client()
.get(getRequest("test").type("type1").id("2").operationThreaded(true))
.actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
getResult =
client()
.get(getRequest("test").type("type1").id("2").operationThreaded(true))
.actionGet(1000);
assertThat(getResult.getSourceAsString(), equalTo(source("2", "test")));
}
}