public static void main(String[] args) throws Exception {
Settings settings = settingsBuilder().put("gateway.type", "none").build();
Node node1 = nodeBuilder().settings(settings).node();
Node node2 = nodeBuilder().settings(settings).node();
final Node client = nodeBuilder().settings(settings).client(true).node();
final int NUMBER_OF_DOCS = 10000;
final int NUMBER_OF_THREADS = 10;
final long NUMBER_OF_ITERATIONS = SizeValue.parseSizeValue("10k").singles();
final long DELETE_EVERY = 10;
final CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS);
Thread[] threads = new Thread[NUMBER_OF_THREADS];
for (int i = 0; i < threads.length; i++) {
threads[i] =
new Thread() {
@Override
public void run() {
try {
for (long i = 0; i < NUMBER_OF_ITERATIONS; i++) {
if ((i % DELETE_EVERY) == 0) {
client
.client()
.prepareDelete(
"test",
"type1",
Integer.toString(ThreadLocalRandom.current().nextInt(NUMBER_OF_DOCS)))
.execute()
.actionGet();
} else {
client
.client()
.prepareIndex(
"test",
"type1",
Integer.toString(ThreadLocalRandom.current().nextInt(NUMBER_OF_DOCS)))
.setSource("field1", "value1")
.execute()
.actionGet();
}
}
} finally {
latch.countDown();
}
}
};
}
for (Thread thread : threads) {
thread.start();
}
latch.await();
System.out.println("done indexing, verifying docs");
client.client().admin().indices().prepareRefresh().execute().actionGet();
for (int i = 0; i < NUMBER_OF_DOCS; i++) {
String id = Integer.toString(i);
for (int j = 0; j < 5; j++) {
SearchResponse response =
client
.client()
.prepareSearch()
.setQuery(QueryBuilders.termQuery("_id", id))
.execute()
.actionGet();
if (response.getHits().totalHits() > 1) {
System.err.println("[" + i + "] FAIL, HITS [" + response.getHits().totalHits() + "]");
}
}
GetResponse getResponse =
client.client().prepareGet("test", "type1", id).execute().actionGet();
if (getResponse.exists()) {
long version = getResponse.version();
for (int j = 0; j < 5; j++) {
getResponse = client.client().prepareGet("test", "type1", id).execute().actionGet();
if (!getResponse.exists()) {
System.err.println("[" + i + "] FAIL, EXISTED, and NOT_EXISTED");
break;
}
if (version != getResponse.version()) {
System.err.println(
"["
+ i
+ "] FAIL, DIFFERENT VERSIONS: ["
+ version
+ "], ["
+ getResponse.version()
+ "]");
break;
}
}
} else {
for (int j = 0; j < 5; j++) {
getResponse = client.client().prepareGet("test", "type1", id).execute().actionGet();
if (getResponse.exists()) {
System.err.println("[" + i + "] FAIL, EXISTED, and NOT_EXISTED");
break;
}
}
}
}
System.out.println("done.");
client.close();
node1.close();
node2.close();
}
public void run() throws Exception {
Node[] nodes = new Node[numberOfNodes];
for (int i = 0; i < nodes.length; i++) {
nodes[i] = NodeBuilder.nodeBuilder().settings(settings).node();
}
client = NodeBuilder.nodeBuilder().settings(settings).client(true).node();
client
.client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(
settingsBuilder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", numberOfReplicas))
.execute()
.actionGet();
logger.info("********** [START] INDEXING INITIAL DOCS");
for (long i = 0; i < initialNumberOfDocs; i++) {
indexDoc();
}
logger.info("********** [DONE ] INDEXING INITIAL DOCS");
Indexer[] indexerThreads = new Indexer[indexers];
for (int i = 0; i < indexerThreads.length; i++) {
indexerThreads[i] = new Indexer();
}
for (int i = 0; i < indexerThreads.length; i++) {
indexerThreads[i].start();
}
long testStart = System.currentTimeMillis();
// start doing the rolling restart
int nodeIndex = 0;
while (true) {
File[] nodeData =
((InternalNode) nodes[nodeIndex])
.injector()
.getInstance(NodeEnvironment.class)
.nodeDataLocations();
nodes[nodeIndex].close();
if (clearNodeData) {
FileSystemUtils.deleteRecursively(nodeData);
}
try {
ClusterHealthResponse clusterHealth =
client
.client()
.admin()
.cluster()
.prepareHealth()
.setWaitForGreenStatus()
.setWaitForNodes(Integer.toString(numberOfNodes + 0 /* client node*/))
.setWaitForRelocatingShards(0)
.setTimeout("10m")
.execute()
.actionGet();
if (clusterHealth.timedOut()) {
logger.warn("timed out waiting for green status....");
}
} catch (Exception e) {
logger.warn("failed to execute cluster health....");
}
nodes[nodeIndex] = NodeBuilder.nodeBuilder().settings(settings).node();
Thread.sleep(1000);
try {
ClusterHealthResponse clusterHealth =
client
.client()
.admin()
.cluster()
.prepareHealth()
.setWaitForGreenStatus()
.setWaitForNodes(Integer.toString(numberOfNodes + 1 /* client node*/))
.setWaitForRelocatingShards(0)
.setTimeout("10m")
.execute()
.actionGet();
if (clusterHealth.timedOut()) {
logger.warn("timed out waiting for green status....");
}
} catch (Exception e) {
logger.warn("failed to execute cluster health....");
}
if (++nodeIndex == nodes.length) {
nodeIndex = 0;
}
if ((System.currentTimeMillis() - testStart) > period.millis()) {
logger.info("test finished");
break;
}
}
for (int i = 0; i < indexerThreads.length; i++) {
indexerThreads[i].close = true;
}
Thread.sleep(indexerThrottle.millis() + 10000);
for (int i = 0; i < indexerThreads.length; i++) {
if (!indexerThreads[i].closed) {
logger.warn("thread not closed!");
}
}
client.client().admin().indices().prepareRefresh().execute().actionGet();
// check the status
IndicesStatusResponse status =
client.client().admin().indices().prepareStatus("test").execute().actionGet();
for (IndexShardStatus shardStatus : status.index("test")) {
ShardStatus shard = shardStatus.shards()[0];
logger.info("shard [{}], docs [{}]", shard.shardId(), shard.getDocs().numDocs());
for (ShardStatus shardStatu : shardStatus) {
if (shard.docs().numDocs() != shardStatu.docs().numDocs()) {
logger.warn(
"shard doc number does not match!, got {} and {}",
shard.docs().numDocs(),
shardStatu.docs().numDocs());
}
}
}
// check the count
for (int i = 0; i < (nodes.length * 5); i++) {
CountResponse count =
client.client().prepareCount().setQuery(matchAllQuery()).execute().actionGet();
logger.info(
"indexed [{}], count [{}], [{}]",
count.count(),
indexCounter.get(),
count.count() == indexCounter.get() ? "OK" : "FAIL");
if (count.count() != indexCounter.get()) {
logger.warn("count does not match!");
}
}
// scan all the docs, verify all have the same version based on the number of replicas
SearchResponse searchResponse =
client
.client()
.prepareSearch()
.setSearchType(SearchType.SCAN)
.setQuery(matchAllQuery())
.setSize(50)
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
logger.info("Verifying versions for {} hits...", searchResponse.hits().totalHits());
while (true) {
searchResponse =
client
.client()
.prepareSearchScroll(searchResponse.scrollId())
.setScroll(TimeValue.timeValueMinutes(2))
.execute()
.actionGet();
if (searchResponse.failedShards() > 0) {
logger.warn("Search Failures " + Arrays.toString(searchResponse.shardFailures()));
}
for (SearchHit hit : searchResponse.hits()) {
long version = -1;
for (int i = 0; i < (numberOfReplicas + 1); i++) {
GetResponse getResponse =
client.client().prepareGet(hit.index(), hit.type(), hit.id()).execute().actionGet();
if (version == -1) {
version = getResponse.version();
} else {
if (version != getResponse.version()) {
logger.warn(
"Doc {} has different version numbers {} and {}",
hit.id(),
version,
getResponse.version());
}
}
}
}
if (searchResponse.hits().hits().length == 0) {
break;
}
}
logger.info("Done verifying versions");
client.close();
for (Node node : nodes) {
node.close();
}
}