@Test
public void testIpMultiField() throws Exception {
assertAcked(
client()
.admin()
.indices()
.prepareCreate("my-index")
.addMapping("my-type", createMappingSource("ip")));
GetMappingsResponse getMappingsResponse =
client().admin().indices().prepareGetMappings("my-index").get();
MappingMetaData mappingMetaData = getMappingsResponse.mappings().get("my-index").get("my-type");
assertThat(mappingMetaData, not(nullValue()));
Map<String, Object> mappingSource = mappingMetaData.sourceAsMap();
Map aField = ((Map) XContentMapValues.extractValue("properties.a", mappingSource));
assertThat(aField.size(), equalTo(2));
assertThat(aField.get("type").toString(), equalTo("ip"));
assertThat(aField.get("fields"), notNullValue());
Map bField = ((Map) XContentMapValues.extractValue("properties.a.fields.b", mappingSource));
assertThat(bField.size(), equalTo(2));
assertThat(bField.get("type").toString(), equalTo("string"));
assertThat(bField.get("index").toString(), equalTo("not_analyzed"));
client()
.prepareIndex("my-index", "my-type", "1")
.setSource("a", "127.0.0.1")
.setRefresh(true)
.get();
CountResponse countResponse =
client().prepareCount("my-index").setQuery(matchQuery("a.b", "127.0.0.1")).get();
assertThat(countResponse.getCount(), equalTo(1l));
}
protected long getCount(final String index, final String type) {
logger.debug("getCount()");
esSetup.client().admin().indices().refresh(new RefreshRequest()).actionGet();
final CountResponse count =
esSetup.client().count(new CountRequest(index).types(type)).actionGet();
return count.getCount();
}
@Override
public IndexStat getIndexStat() {
CountRequestBuilder countRequest =
client
.prepareCount(this.getIndexName())
.setTypes(this.getIndexType())
.setQuery(QueryBuilders.matchAllQuery());
CountResponse response = countRequest.get();
return new IndexStat(getLastSynchronization(), response.getCount());
}
@Test
public void simpleCountEarlyTerminationTests() throws Exception {
// set up one shard only to test early termination
prepareCreate("test")
.setSettings(
SETTING_NUMBER_OF_SHARDS, 1,
SETTING_NUMBER_OF_REPLICAS, 0)
.get();
ensureGreen();
int max = randomIntBetween(3, 29);
List<IndexRequestBuilder> docbuilders = new ArrayList<>(max);
for (int i = 1; i <= max; i++) {
String id = String.valueOf(i);
docbuilders.add(client().prepareIndex("test", "type1", id).setSource("field", i));
}
indexRandom(true, docbuilders);
ensureGreen();
refresh();
// sanity check
CountResponse countResponse =
client()
.prepareCount("test")
.setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(max))
.execute()
.actionGet();
assertHitCount(countResponse, max);
// threshold <= actual count
for (int i = 1; i <= max; i++) {
countResponse =
client()
.prepareCount("test")
.setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(max))
.setTerminateAfter(i)
.execute()
.actionGet();
assertHitCount(countResponse, i);
assertTrue(countResponse.terminatedEarly());
}
// threshold > actual count
countResponse =
client()
.prepareCount("test")
.setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(max))
.setTerminateAfter(max + randomIntBetween(1, max))
.execute()
.actionGet();
assertHitCount(countResponse, max);
assertFalse(countResponse.terminatedEarly());
}
@Test
public void dynamicUpdates() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
int recCount = randomIntBetween(200, 600);
int numberOfTypes = randomIntBetween(1, 5);
List<IndexRequestBuilder> indexRequests = Lists.newArrayList();
for (int rec = 0; rec < recCount; rec++) {
String type = "type" + (rec % numberOfTypes);
String fieldName = "field_" + type + "_" + rec;
indexRequests.add(
client()
.prepareIndex("test", type, Integer.toString(rec))
.setSource(fieldName, "some_value"));
}
indexRandom(true, indexRequests);
logger.info("checking all the documents are there");
RefreshResponse refreshResponse =
client().admin().indices().prepareRefresh().execute().actionGet();
assertThat(refreshResponse.getFailedShards(), equalTo(0));
CountResponse response = client().prepareCount("test").execute().actionGet();
assertThat(response.getCount(), equalTo((long) recCount));
logger.info("checking all the fields are in the mappings");
for (int rec = 0; rec < recCount; rec++) {
String type = "type" + (rec % numberOfTypes);
String fieldName = "field_" + type + "_" + rec;
assertConcreteMappingsOnAll("test", type, fieldName);
}
}
@Test
public void connectRiverAndSendMessages() throws InterruptedException {
Thread.sleep(1000);
String field = "content";
String msg = "sammy";
logger.debug("Publishing message [channel={}, msg={}]", channel, msg);
jedis.publish(channel, msg);
Thread.sleep(1000);
refreshIndex();
QuerySourceBuilder builder = new QuerySourceBuilder();
builder.setQuery(queryString(field + ":" + msg));
logger.debug(
"Counting [index={}, type={}, field={}, msg={}]",
new Object[] {index, channel, field, msg});
CountResponse resp =
node.client().count(countRequest(index).types(channel).source(builder)).actionGet();
assertEquals(1, resp.getCount());
msg = "coldplay";
logger.debug(
"Counting [index={}, type={}, field={}, msg={}]",
new Object[] {index, channel, field, msg});
resp = node.client().count(countRequest(index).types(channel).source(builder)).actionGet();
assertEquals(0, resp.getCount());
logger.debug("Publishing message [channel={}]", channel);
jedis.publish(channel, msg);
Thread.sleep(1000);
refreshIndex();
logger.debug(
"Counting [index={}, type={}, field={}, msg={}]",
new Object[] {index, channel, field, msg});
resp = node.client().count(countRequest(index).types(channel).source(builder)).actionGet();
assertEquals(1, resp.getCount());
shutdown();
}
@Test
public void testIgnoreScript() throws Throwable {
logger.debug("Start testIgnoreScript");
try {
logger.debug("Create river {}", getRiver());
String script = "ctx.ignore = true;";
super.createRiver(
"/test/elasticsearch/plugin/river/mongodb/script/test-mongodb-river-with-script.json",
getRiver(),
String.valueOf(getMongoPort1()),
String.valueOf(getMongoPort2()),
String.valueOf(getMongoPort3()),
getDatabase(),
getCollection(),
script,
getIndex(),
getDatabase());
String mongoDocument =
copyToStringFromClasspath(
"/test/elasticsearch/plugin/river/mongodb/script/test-simple-mongodb-document.json");
DBObject dbObject = (DBObject) JSON.parse(mongoDocument);
WriteResult result = mongoCollection.insert(dbObject);
Thread.sleep(wait);
logger.info("WriteResult: {}", result.toString());
refreshIndex();
ActionFuture<IndicesExistsResponse> response =
getNode().client().admin().indices().exists(new IndicesExistsRequest(getIndex()));
assertThat(response.actionGet().isExists(), equalTo(true));
CountResponse countResponse = getNode().client().count(countRequest(getIndex())).actionGet();
logger.info("Document count: {}", countResponse.getCount());
assertThat(countResponse.getCount(), equalTo(0l));
mongoCollection.remove(dbObject);
} catch (Throwable t) {
logger.error("testIgnoreScript failed.", t);
t.printStackTrace();
throw t;
} finally {
super.deleteRiver();
super.deleteIndex();
}
}
@Test
public void testImportAttachment() throws Exception {
logger.debug("*** testImportAttachment ***");
byte[] content =
copyToBytesFromClasspath(
"/test/elasticsearch/plugin/river/mongodb/gridfs/test-attachment.html");
logger.debug("Content in bytes: {}", content.length);
GridFS gridFS = new GridFS(mongoDB);
GridFSInputFile in = gridFS.createFile(content);
in.setFilename("test-attachment.html");
in.setContentType("text/html");
in.save();
in.validate();
String id = in.getId().toString();
logger.debug("GridFS in: {}", in);
logger.debug("Document created with id: {}", id);
GridFSDBFile out = gridFS.findOne(in.getFilename());
logger.debug("GridFS from findOne: {}", out);
out = gridFS.findOne(new ObjectId(id));
logger.debug("GridFS from findOne: {}", out);
Assert.assertEquals(out.getId(), in.getId());
Thread.sleep(wait);
refreshIndex();
CountResponse countResponse = getNode().client().count(countRequest(getIndex())).actionGet();
logger.debug("Index total count: {}", countResponse.getCount());
assertThat(countResponse.getCount(), equalTo(1l));
countResponse =
getNode().client().count(countRequest(getIndex()).query(fieldQuery("_id", id))).actionGet();
logger.debug("Index count for id {}: {}", id, countResponse.getCount());
assertThat(countResponse.getCount(), equalTo(1l));
SearchResponse response =
getNode()
.client()
.prepareSearch(getIndex())
.setQuery(QueryBuilders.queryString("Aliquam"))
.execute()
.actionGet();
logger.debug("SearchResponse {}", response.toString());
long totalHits = response.getHits().getTotalHits();
logger.debug("TotalHits: {}", totalHits);
assertThat(totalHits, equalTo(1l));
gridFS.remove(new ObjectId(id));
Thread.sleep(wait);
refreshIndex();
countResponse =
getNode().client().count(countRequest(getIndex()).query(fieldQuery("_id", id))).actionGet();
logger.debug("Count after delete request: {}", countResponse.getCount());
assertThat(countResponse.getCount(), equalTo(0L));
}
@Test
public void testEsIndexBolt() throws Exception {
String index = "index1";
String type = "type1";
Tuple tuple = createTestTuple(index, type);
bolt.execute(tuple);
verify(outputCollector).ack(tuple);
node.client().admin().indices().prepareRefresh(index).execute().actionGet();
CountResponse resp =
node.client()
.prepareCount(index)
.setQuery(new TermQueryBuilder("_type", type))
.execute()
.actionGet();
Assert.assertEquals(1, resp.getCount());
}
@Test
public void dynamicUpdates() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
ImmutableSettings.settingsBuilder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 0))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
long recCount = 20;
for (int rec = 0; rec < recCount; rec++) {
client()
.prepareIndex("test", "type", "rec" + rec)
.setSource("field" + rec, "some_value")
.execute()
.actionGet();
}
RefreshResponse refreshResponse =
client().admin().indices().prepareRefresh().execute().actionGet();
assertThat(refreshResponse.getFailedShards(), equalTo(0));
logger.info("Searching");
CountResponse response = client().prepareCount("test").execute().actionGet();
assertThat(response.getCount(), equalTo(recCount));
}
@Test
public void testNoMasterActions_writeMasterBlock() throws Exception {
Settings settings =
settingsBuilder()
.put("discovery.type", "zen")
.put("action.auto_create_index", false)
.put("discovery.zen.minimum_master_nodes", 2)
.put("discovery.zen.ping_timeout", "200ms")
.put("discovery.initial_state_timeout", "500ms")
.put(DiscoverySettings.NO_MASTER_BLOCK, "write")
.build();
internalCluster().startNode(settings);
// start a second node, create an index, and then shut it down so we have no master block
internalCluster().startNode(settings);
prepareCreate("test1").setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).get();
prepareCreate("test2")
.setSettings(
IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.get();
client().admin().cluster().prepareHealth("_all").setWaitForGreenStatus().get();
client().prepareIndex("test1", "type1", "1").setSource("field", "value1").get();
client().prepareIndex("test2", "type1", "1").setSource("field", "value1").get();
refresh();
ensureSearchable("test1", "test2");
ClusterStateResponse clusterState = client().admin().cluster().prepareState().get();
logger.info("Cluster state:\n" + clusterState.getState().prettyPrint());
internalCluster().stopRandomDataNode();
assertThat(
awaitBusy(
new Predicate<Object>() {
public boolean apply(Object o) {
ClusterState state =
client().admin().cluster().prepareState().setLocal(true).get().getState();
return state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID);
}
}),
equalTo(true));
GetResponse getResponse = client().prepareGet("test1", "type1", "1").get();
assertExists(getResponse);
CountResponse countResponse = client().prepareCount("test1").get();
assertHitCount(countResponse, 1l);
SearchResponse searchResponse = client().prepareSearch("test1").get();
assertHitCount(searchResponse, 1l);
countResponse = client().prepareCount("test2").get();
assertThat(countResponse.getTotalShards(), equalTo(2));
assertThat(countResponse.getSuccessfulShards(), equalTo(1));
TimeValue timeout = TimeValue.timeValueMillis(200);
long now = System.currentTimeMillis();
try {
client()
.prepareUpdate("test1", "type1", "1")
.setDoc("field", "value2")
.setTimeout(timeout)
.get();
fail("Expected ClusterBlockException");
} catch (ClusterBlockException e) {
assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
}
now = System.currentTimeMillis();
try {
client()
.prepareIndex("test1", "type1", "1")
.setSource(XContentFactory.jsonBuilder().startObject().endObject())
.setTimeout(timeout)
.get();
fail("Expected ClusterBlockException");
} catch (ClusterBlockException e) {
assertThat(System.currentTimeMillis() - now, greaterThan(timeout.millis() - 50));
assertThat(e.status(), equalTo(RestStatus.SERVICE_UNAVAILABLE));
}
internalCluster().startNode(settings);
client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").get();
}
@Test
public void testDeleteDocument() throws Throwable {
logger.debug("Start testDeleteDocument");
try {
logger.debug("Create river {}", getRiver());
String script = "if (ctx.document.to_be_deleted == true) { ctx.operation = 'd' };";
super.createRiver(
"/test/elasticsearch/plugin/river/mongodb/script/test-mongodb-river-with-script.json",
getRiver(),
String.valueOf(getMongoPort1()),
String.valueOf(getMongoPort2()),
String.valueOf(getMongoPort3()),
getDatabase(),
getCollection(),
script,
getIndex(),
getDatabase());
String mongoDocument =
copyToStringFromClasspath(
"/test/elasticsearch/plugin/river/mongodb/script/test-simple-mongodb-document.json");
DBObject dbObject = (DBObject) JSON.parse(mongoDocument);
WriteResult result = mongoCollection.insert(dbObject);
Thread.sleep(wait);
String id = dbObject.get("_id").toString();
logger.info("WriteResult: {}", result.toString());
refreshIndex();
ActionFuture<IndicesExistsResponse> response =
getNode().client().admin().indices().exists(new IndicesExistsRequest(getIndex()));
assertThat(response.actionGet().isExists(), equalTo(true));
SearchResponse sr =
getNode()
.client()
.prepareSearch(getIndex())
.setQuery(fieldQuery("_id", id))
.execute()
.actionGet();
logger.debug("SearchResponse {}", sr.toString());
long totalHits = sr.getHits().getTotalHits();
logger.debug("TotalHits: {}", totalHits);
assertThat(totalHits, equalTo(1l));
dbObject.put("to_be_deleted", Boolean.TRUE);
mongoCollection.save(dbObject);
Thread.sleep(wait);
refreshIndex();
CountResponse countResponse = getNode().client().count(countRequest(getIndex())).actionGet();
logger.info("Document count: {}", countResponse.getCount());
assertThat(countResponse.getCount(), equalTo(0l));
mongoCollection.remove(dbObject);
} catch (Throwable t) {
logger.error("testDeleteDocument failed.", t);
t.printStackTrace();
throw t;
} finally {
super.deleteRiver();
super.deleteIndex();
}
}
public long countAll(String... indices) {
CountResponse response =
client.prepareCount(indices).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet();
return response.getCount();
}
public void run() throws Exception {
Node[] nodes = new Node[numberOfNodes];
for (int i = 0; i < nodes.length; i++) {
nodes[i] = NodeBuilder.nodeBuilder().settings(settings).node();
}
client = NodeBuilder.nodeBuilder().settings(settings).client(true).node();
client
.client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
settingsBuilder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", numberOfReplicas))
.execute()
.actionGet();
logger.info("********** [START] INDEXING INITIAL DOCS");
for (long i = 0; i < initialNumberOfDocs; i++) {
indexDoc();
}
logger.info("********** [DONE ] INDEXING INITIAL DOCS");
Indexer[] indexerThreads = new Indexer[indexers];
for (int i = 0; i < indexerThreads.length; i++) {
indexerThreads[i] = new Indexer();
}
for (int i = 0; i < indexerThreads.length; i++) {
indexerThreads[i].start();
}
long testStart = System.currentTimeMillis();
// start doing the rolling restart
int nodeIndex = 0;
while (true) {
File[] nodeData =
((InternalNode) nodes[nodeIndex])
.injector()
.getInstance(NodeEnvironment.class)
.nodeDataLocations();
nodes[nodeIndex].close();
if (clearNodeData) {
FileSystemUtils.deleteRecursively(nodeData);
}
try {
ClusterHealthResponse clusterHealth =
client
.client()
.admin()
.cluster()
.prepareHealth()
.setWaitForGreenStatus()
.setWaitForNodes(Integer.toString(numberOfNodes + 0 /* client node*/))
.setWaitForRelocatingShards(0)
.setTimeout("10m")
.execute()
.actionGet();
if (clusterHealth.timedOut()) {
logger.warn("timed out waiting for green status....");
}
} catch (Exception e) {
logger.warn("failed to execute cluster health....");
}
nodes[nodeIndex] = NodeBuilder.nodeBuilder().settings(settings).node();
Thread.sleep(1000);
try {
ClusterHealthResponse clusterHealth =
client
.client()
.admin()
.cluster()
.prepareHealth()
.setWaitForGreenStatus()
.setWaitForNodes(Integer.toString(numberOfNodes + 1 /* client node*/))
.setWaitForRelocatingShards(0)
.setTimeout("10m")
.execute()
.actionGet();
if (clusterHealth.timedOut()) {
logger.warn("timed out waiting for green status....");
}
} catch (Exception e) {
logger.warn("failed to execute cluster health....");
}
if (++nodeIndex == nodes.length) {
nodeIndex = 0;
}
if ((System.currentTimeMillis() - testStart) > period.millis()) {
logger.info("test finished");
break;
}
}
for (int i = 0; i < indexerThreads.length; i++) {
indexerThreads[i].close = true;
}
Thread.sleep(indexerThrottle.millis() + 10000);
for (int i = 0; i < indexerThreads.length; i++) {
if (!indexerThreads[i].closed) {
logger.warn("thread not closed!");
}
}
client.client().admin().indices().prepareRefresh().execute().actionGet();
// check the status
IndicesStatusResponse status =
client.client().admin().indices().prepareStatus("test").execute().actionGet();
for (IndexShardStatus shardStatus : status.index("test")) {
ShardStatus shard = shardStatus.shards()[0];
logger.info("shard [{}], docs [{}]", shard.shardId(), shard.getDocs().numDocs());
for (ShardStatus shardStatu : shardStatus) {
if (shard.docs().numDocs() != shardStatu.docs().numDocs()) {
logger.warn(
"shard doc number does not match!, got {} and {}",
shard.docs().numDocs(),
shardStatu.docs().numDocs());
}
}
}
// check the count
for (int i = 0; i < (nodes.length * 5); i++) {
CountResponse count =
client.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet();
logger.info(
"indexed [{}], count [{}], [{}]",
count.count(),
indexCounter.get(),
count.count() == indexCounter.get() ? "OK" : "FAIL");
if (count.count() != indexCounter.get()) {
logger.warn("count does not match!");
}
}
// scan all the docs, verify all have the same version based on the number of replicas
SearchResponse searchResponse =
client
.client()
.prepareSearch()
.setSearchType(SearchType.SCAN)
.setQuery(matchAllQuery())
.setSize(50)
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
logger.info("Verifying versions for {} hits...", searchResponse.hits().totalHits());
while (true) {
searchResponse =
client
.client()
.prepareSearchScroll(searchResponse.scrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
if (searchResponse.failedShards() > 0) {
logger.warn("Search Failures " + Arrays.toString(searchResponse.shardFailures()));
}
for (SearchHit hit : searchResponse.hits()) {
long version = -1;
for (int i = 0; i < (numberOfReplicas + 1); i++) {
GetResponse getResponse =
client.client().prepareGet(hit.index(), hit.type(), hit.id()).execute().actionGet();
if (version == -1) {
version = getResponse.version();
} else {
if (version != getResponse.version()) {
logger.warn(
"Doc {} has different version numbers {} and {}",
hit.id(),
version,
getResponse.version());
}
}
}
}
if (searchResponse.hits().hits().length == 0) {
break;
}
}
logger.info("Done verifying versions");
client.close();
for (Node node : nodes) {
node.close();
}
}
private boolean isAnyIndexPresent() {
CountResponse numberOfElements = client.prepareCount().execute().actionGet();
return numberOfElements.getCount() > 0;
}
@Test
public void dynamicUpdates() throws Exception {
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
ImmutableSettings.settingsBuilder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0))
.execute()
.actionGet();
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.setWaitForGreenStatus()
.execute()
.actionGet();
int recCount = randomIntBetween(200, 600);
int numberOfTypes = randomIntBetween(1, 5);
List<IndexRequestBuilder> indexRequests = Lists.newArrayList();
for (int rec = 0; rec < recCount; rec++) {
String type = "type" + (rec % numberOfTypes);
String fieldName = "field_" + type + "_" + rec;
indexRequests.add(
client()
.prepareIndex("test", type, Integer.toString(rec))
.setSource(fieldName, "some_value"));
}
indexRandom(true, indexRequests);
logger.info("checking all the documents are there");
RefreshResponse refreshResponse =
client().admin().indices().prepareRefresh().execute().actionGet();
assertThat(refreshResponse.getFailedShards(), equalTo(0));
CountResponse response = client().prepareCount("test").execute().actionGet();
assertThat(response.getCount(), equalTo((long) recCount));
logger.info("checking all the fields are in the mappings");
reRunTest:
while (true) {
Map<String, String> typeToSource = Maps.newHashMap();
ClusterState state = client().admin().cluster().prepareState().get().getState();
for (ObjectObjectCursor<String, MappingMetaData> cursor :
state.getMetaData().getIndices().get("test").getMappings()) {
typeToSource.put(cursor.key, cursor.value.source().string());
}
for (int rec = 0; rec < recCount; rec++) {
String type = "type" + (rec % numberOfTypes);
String fieldName = "field_" + type + "_" + rec;
fieldName = "\"" + fieldName + "\""; // quote it, so we make sure we catch the exact one
if (!typeToSource.containsKey(type) || !typeToSource.get(type).contains(fieldName)) {
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.execute()
.actionGet();
awaitBusy(
new Predicate<Object>() {
@Override
public boolean apply(Object input) {
PendingClusterTasksResponse pendingTasks =
client().admin().cluster().preparePendingClusterTasks().get();
return pendingTasks.pendingTasks().isEmpty();
}
});
client()
.admin()
.cluster()
.prepareHealth()
.setWaitForEvents(Priority.LANGUID)
.execute()
.actionGet();
// its going to break, before we do, make sure that the cluster state hasn't changed on
// us...
ClusterState state2 = client().admin().cluster().prepareState().get().getState();
if (state.version() != state2.version()) {
logger.info(
"not the same version, used for test {}, new one {}, re-running test, first wait for mapping to wait",
state.version(),
state2.version());
continue reRunTest;
}
logger.info(
"failing, type {}, field {}, mapping {}", type, fieldName, typeToSource.get(type));
assertThat(typeToSource.get(type), containsString(fieldName));
}
}
break;
}
}
public static void main(String[] args) throws Exception {
final int NUM_NODES = 40;
final int NUM_INDICES = 100;
final int NUM_DOCS = 2;
final int FLUSH_AFTER = 1;
final Settings nodeSettings =
Settings.settingsBuilder()
.put("transport.netty.connections_per_node.low", 0)
.put("transport.netty.connections_per_node.med", 0)
.put("transport.netty.connections_per_node.high", 1)
.build();
final Settings indexSettings =
Settings.settingsBuilder().put("index.number_of_shards", 1).build();
List<Node> nodes = Lists.newArrayList();
for (int i = 0; i < NUM_NODES; i++) {
nodes.add(
NodeBuilder.nodeBuilder()
.settings(Settings.settingsBuilder().put(nodeSettings).put("name", "node" + i))
.node());
}
Client client = nodes.get(0).client();
for (int index = 0; index < NUM_INDICES; index++) {
String indexName = "index_" + index;
System.out.println("--> Processing index [" + indexName + "]...");
client
.admin()
.indices()
.prepareCreate(indexName)
.setSettings(indexSettings)
.execute()
.actionGet();
boolean flushed = false;
for (int doc = 0; doc < NUM_DOCS; doc++) {
if (!flushed && doc > FLUSH_AFTER) {
flushed = true;
client.admin().indices().prepareFlush(indexName).execute().actionGet();
}
client
.prepareIndex(indexName, "type1", Integer.toString(doc))
.setSource("field", "value" + doc)
.execute()
.actionGet();
}
System.out.println("--> DONE index [" + indexName + "]");
}
System.out.println("--> Initiating shutdown");
for (Node node : nodes) {
node.close();
}
System.out.println("--> Waiting for all nodes to be closed...");
while (true) {
boolean allAreClosed = true;
for (Node node : nodes) {
if (!node.isClosed()) {
allAreClosed = false;
break;
}
}
if (allAreClosed) {
break;
}
Thread.sleep(100);
}
System.out.println("Waiting a bit for node lock to really be released?");
Thread.sleep(5000);
System.out.println("--> All nodes are closed, starting back...");
nodes = Lists.newArrayList();
for (int i = 0; i < NUM_NODES; i++) {
nodes.add(
NodeBuilder.nodeBuilder()
.settings(Settings.settingsBuilder().put(nodeSettings).put("name", "node" + i))
.node());
}
client = nodes.get(0).client();
System.out.println("--> Waiting for green status");
while (true) {
ClusterHealthResponse clusterHealth =
client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
if (clusterHealth.isTimedOut()) {
System.err.println(
"--> cluster health timed out..., active shards ["
+ clusterHealth.getActiveShards()
+ "]");
} else {
break;
}
}
System.out.println("Verifying counts...");
for (int index = 0; index < NUM_INDICES; index++) {
String indexName = "index_" + index;
CountResponse count =
client
.prepareCount(indexName)
.setQuery(QueryBuilders.matchAllQuery())
.execute()
.actionGet();
if (count.getCount() != NUM_DOCS) {
System.err.println(
"Wrong count value, expected ["
+ NUM_DOCS
+ "], got ["
+ count.getCount()
+ "] for index ["
+ indexName
+ "]");
}
}
System.out.println("Test end");
for (Node node : nodes) {
node.close();
}
}
/**
* If you need to run specific tests, just override this method. By default, we check the number
* of expected documents
*
* @param node Elasticsearch current node
*/
protected void postInjectionTests(Node node) {
CountResponse response = node.client().prepareCount("test").execute().actionGet();
// We have consumed all messages. We can now check expected number of documents
Assert.assertEquals(
"Wrong number of documents found", expectedDocuments(), response.getCount());
}
@Test
public void testBroadcastOperations() throws IOException {
startNode("server1");
client("server1").admin().indices().prepareCreate("test").execute().actionGet(5000);
logger.info("Running Cluster Health");
ClusterHealthResponse clusterHealth =
client("server1")
.admin()
.cluster()
.health(clusterHealthRequest().waitForYellowStatus())
.actionGet();
logger.info("Done Cluster Health, status " + clusterHealth.status());
assertThat(clusterHealth.timedOut(), equalTo(false));
assertThat(clusterHealth.status(), equalTo(ClusterHealthStatus.YELLOW));
client("server1")
.index(indexRequest("test").type("type1").id("1").source(source("1", "test")))
.actionGet();
FlushResponse flushResponse =
client("server1").admin().indices().flush(flushRequest("test")).actionGet();
assertThat(flushResponse.totalShards(), equalTo(10));
assertThat(flushResponse.successfulShards(), equalTo(5));
assertThat(flushResponse.failedShards(), equalTo(0));
client("server1")
.index(indexRequest("test").type("type1").id("2").source(source("2", "test")))
.actionGet();
RefreshResponse refreshResponse =
client("server1").admin().indices().refresh(refreshRequest("test")).actionGet();
assertThat(refreshResponse.totalShards(), equalTo(10));
assertThat(refreshResponse.successfulShards(), equalTo(5));
assertThat(refreshResponse.failedShards(), equalTo(0));
logger.info("Count");
// check count
for (int i = 0; i < 5; i++) {
// test successful
CountResponse countResponse =
client("server1")
.count(
countRequest("test")
.query(termQuery("_type", "type1"))
.operationThreading(BroadcastOperationThreading.NO_THREADS))
.actionGet();
assertThat(countResponse.count(), equalTo(2l));
assertThat(countResponse.totalShards(), equalTo(5));
assertThat(countResponse.successfulShards(), equalTo(5));
assertThat(countResponse.failedShards(), equalTo(0));
}
for (int i = 0; i < 5; i++) {
CountResponse countResponse =
client("server1")
.count(
countRequest("test")
.query(termQuery("_type", "type1"))
.operationThreading(BroadcastOperationThreading.SINGLE_THREAD))
.actionGet();
assertThat(countResponse.count(), equalTo(2l));
assertThat(countResponse.totalShards(), equalTo(5));
assertThat(countResponse.successfulShards(), equalTo(5));
assertThat(countResponse.failedShards(), equalTo(0));
}
for (int i = 0; i < 5; i++) {
CountResponse countResponse =
client("server1")
.count(
countRequest("test")
.query(termQuery("_type", "type1"))
.operationThreading(BroadcastOperationThreading.THREAD_PER_SHARD))
.actionGet();
assertThat(countResponse.count(), equalTo(2l));
assertThat(countResponse.totalShards(), equalTo(5));
assertThat(countResponse.successfulShards(), equalTo(5));
assertThat(countResponse.failedShards(), equalTo(0));
}
for (int i = 0; i < 5; i++) {
// test failed (simply query that can't be parsed)
CountResponse countResponse =
client("server1")
.count(
countRequest("test")
.query(Unicode.fromStringAsBytes("{ term : { _type : \"type1 } }")))
.actionGet();
assertThat(countResponse.count(), equalTo(0l));
assertThat(countResponse.totalShards(), equalTo(5));
assertThat(countResponse.successfulShards(), equalTo(0));
assertThat(countResponse.failedShards(), equalTo(5));
for (ShardOperationFailedException exp : countResponse.shardFailures()) {
assertThat(exp.reason(), containsString("QueryParsingException"));
}
}
}
@Test
public void testImportAttachment() throws Exception {
logger.debug("*** testImportAttachment ***");
try {
// createDatabase();
byte[] content = copyToBytesFromClasspath(TEST_ATTACHMENT_HTML);
logger.debug("Content in bytes: {}", content.length);
GridFS gridFS = new GridFS(mongoDB);
GridFSInputFile in = gridFS.createFile(content);
in.setFilename("test-attachment.html");
in.setContentType("text/html");
in.save();
in.validate();
String id = in.getId().toString();
logger.debug("GridFS in: {}", in);
logger.debug("Document created with id: {}", id);
GridFSDBFile out = gridFS.findOne(in.getFilename());
logger.debug("GridFS from findOne: {}", out);
out = gridFS.findOne(new ObjectId(id));
logger.debug("GridFS from findOne: {}", out);
Assert.assertEquals(out.getId(), in.getId());
Thread.sleep(wait);
refreshIndex();
CountResponse countResponse = getNode().client().count(countRequest(getIndex())).actionGet();
logger.debug("Index total count: {}", countResponse.getCount());
assertThat(countResponse.getCount(), equalTo(1l));
GetResponse getResponse = getNode().client().get(getRequest(getIndex()).id(id)).get();
logger.debug("Get request for id {}: {}", id, getResponse.isExists());
assertThat(getResponse.isExists(), equalTo(true));
// countResponse =
// getNode().client().count(countRequest(getIndex()).query(fieldQuery("_id",
// id))).actionGet();
// logger.debug("Index count for id {}: {}", id, countResponse.getCount());
// assertThat(countResponse.getCount(), equalTo(1l));
SearchResponse response =
getNode()
.client()
.prepareSearch(getIndex())
.setQuery(QueryBuilders.queryString("Aliquam"))
.execute()
.actionGet();
logger.debug("SearchResponse {}", response.toString());
long totalHits = response.getHits().getTotalHits();
logger.debug("TotalHits: {}", totalHits);
assertThat(totalHits, equalTo(1l));
gridFS.remove(new ObjectId(id));
Thread.sleep(wait);
refreshIndex();
getResponse = getNode().client().get(getRequest(getIndex()).id(id)).get();
logger.debug("Get request for id {}: {}", id, getResponse.isExists());
assertThat(getResponse.isExists(), equalTo(false));
// countResponse =
// getNode().client().count(countRequest(getIndex()).query(fieldQuery("_id",
// id))).actionGet();
// logger.debug("Count after delete request: {}", countResponse.getCount());
// assertThat(countResponse.getCount(), equalTo(0L));
} catch (Throwable t) {
logger.error("testImportAttachment failed.", t);
Assert.fail("testImportAttachment failed", t);
} finally {
// cleanUp();
}
}
@Test
public void test_all_messages_are_consumed() throws Exception {
// We try to connect to RabbitMQ.
// If it's not launched, we don't fail the test but only log it
try {
ConnectionFactory cfconn = new ConnectionFactory();
cfconn.setHost("localhost");
cfconn.setPort(AMQP.PROTOCOL.PORT);
Connection conn = cfconn.newConnection();
Channel ch = conn.createChannel();
ch.exchangeDeclare("elasticsearch", "direct", true);
AMQP.Queue.DeclareOk queue = ch.queueDeclare("elasticsearch", true, false, false, null);
// We purge the queue in case of something is remaining there
ch.queuePurge("elasticsearch");
pushMessages(ch);
// We can now create our node and our river
Settings settings =
ImmutableSettings.settingsBuilder()
.put("gateway.type", "none")
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.put(nodeSettings())
.build();
node = NodeBuilder.nodeBuilder().local(true).settings(settings).node();
// We first remove existing index if any
try {
node.client().admin().indices().prepareDelete(INDEX).execute().actionGet();
} catch (IndexMissingException e) {
// Index is missing? It's perfectly fine!
}
// Let's create an index for our docs and we will disable refresh
node.client().admin().indices().prepareCreate(INDEX).execute().actionGet();
node.client()
.prepareIndex("_river", "test", "_meta")
.setSource(river())
.execute()
.actionGet();
// We need at some point to check if we have consumed the river
int steps = timeout();
long count = 0;
while (true) {
// We wait for one second
Thread.sleep(1000);
CountResponse response = node.client().prepareCount("test").execute().actionGet();
count = response.getCount();
steps--;
if (steps < 0 || count == expectedDocuments()) {
break;
}
}
ch.close();
conn.close();
postInjectionTests(node);
} catch (ConnectException e) {
logger.warn(
"RabbitMQ service is not launched on localhost:{}. Can not start Integration test. "
+ "Launch `rabbitmq-server`.",
AMQP.PROTOCOL.PORT);
}
}
