@Test
public void testSubmitBenchmark() throws Exception {
final int iters =
between(1, 3); // we run this more than once to make sure metadata is cleaned up propperly
for (int i = 0; i < iters; i++) {
final BenchmarkRequest request =
BenchmarkTestUtil.randomRequest(
client(), indices, numExecutorNodes, competitionSettingsMap);
logger.info(
"--> Submitting benchmark - competitors [{}] iterations [{}]",
request.competitors().size(),
request.settings().iterations());
final BenchmarkResponse response = client().bench(request).actionGet();
assertThat(response, notNullValue());
assertThat(response.state(), equalTo(BenchmarkResponse.State.COMPLETE));
assertFalse(response.hasErrors());
assertThat(response.benchmarkName(), equalTo(BENCHMARK_NAME));
assertThat(response.competitionResults().size(), equalTo(request.competitors().size()));
for (CompetitionResult result : response.competitionResults().values()) {
assertThat(result.nodeResults().size(), equalTo(numExecutorNodes));
validateCompetitionResult(
result, competitionSettingsMap.get(result.competitionName()), true);
}
}
}
@Test
public void testBenchmarkWithErrors() {
List<SearchRequest> reqList = new ArrayList<>();
int numQueries = scaledRandomIntBetween(20, 100);
int numErrors = scaledRandomIntBetween(1, numQueries);
final boolean containsFatal = randomBoolean();
if (containsFatal) {
ScriptScoreFunctionBuilder scriptFunction =
scriptFunction("DOES NOT COMPILE - fails on any shard");
SearchRequest searchRequest =
searchRequest()
.source(
searchSource()
.query(functionScoreQuery(FilterBuilders.matchAllFilter(), scriptFunction)));
reqList.add(searchRequest);
}
for (int i = 0; reqList.size() < numErrors; i++) {
ScriptScoreFunctionBuilder scriptFunction = scriptFunction("throw new RuntimeException();");
SearchRequest searchRequest =
searchRequest()
.source(
searchSource()
.query(functionScoreQuery(FilterBuilders.matchAllFilter(), scriptFunction)));
reqList.add(searchRequest);
}
logger.info("--> run with [{}] errors ", numErrors);
for (int i = 0; reqList.size() < numQueries; i++) {
reqList.add(BenchmarkTestUtil.randomSearch(client(), indices));
}
Collections.shuffle(reqList, getRandom());
final BenchmarkRequest request =
BenchmarkTestUtil.randomRequest(
client(),
indices,
numExecutorNodes,
competitionSettingsMap,
reqList.toArray(new SearchRequest[0]));
logger.info(
"--> Submitting benchmark - competitors [{}] iterations [{}]",
request.competitors().size(),
request.settings().iterations());
final BenchmarkResponse response = client().bench(request).actionGet();
assertThat(response, notNullValue());
if (response.hasErrors() || containsFatal) {
assertThat(response.state(), equalTo(BenchmarkResponse.State.FAILED));
} else {
assertThat(response.state(), equalTo(BenchmarkResponse.State.COMPLETE));
for (CompetitionResult result : response.competitionResults().values()) {
assertThat(result.nodeResults().size(), equalTo(numExecutorNodes));
validateCompetitionResult(
result, competitionSettingsMap.get(result.competitionName()), true);
}
}
assertThat(response.benchmarkName(), equalTo(BENCHMARK_NAME));
}
@Test
public void testListBenchmarks() throws Exception {
SearchRequest searchRequest = prepareBlockingScriptQuery();
final BenchmarkRequest request =
BenchmarkTestUtil.randomRequest(
client(), indices, numExecutorNodes, competitionSettingsMap, searchRequest);
logger.info(
"--> Submitting benchmark - competitors [{}] iterations [{}]",
request.competitors().size(),
request.settings().iterations());
final ActionFuture<BenchmarkResponse> future = client().bench(request);
try {
waitForQuery.await();
final BenchmarkStatusResponse statusResponse =
client().prepareBenchStatus().execute().actionGet();
waitForTestLatch.countDown();
assertThat(statusResponse.benchmarkResponses().size(), equalTo(1));
for (BenchmarkResponse benchmarkResponse : statusResponse.benchmarkResponses()) {
assertThat(benchmarkResponse.benchmarkName(), equalTo(BENCHMARK_NAME));
assertThat(benchmarkResponse.state(), equalTo(BenchmarkResponse.State.RUNNING));
assertFalse(benchmarkResponse.hasErrors());
for (CompetitionResult result : benchmarkResponse.competitionResults().values()) {
assertThat(result.nodeResults().size(), lessThanOrEqualTo(numExecutorNodes));
validateCompetitionResult(
result, competitionSettingsMap.get(result.competitionName()), false);
}
}
} finally {
if (waitForTestLatch.getCount() == 1) {
waitForTestLatch.countDown();
}
client().prepareAbortBench(BENCHMARK_NAME).get();
// Confirm that there are no active benchmarks in the cluster
assertThat(
client().prepareBenchStatus().execute().actionGet().totalActiveBenchmarks(), equalTo(0));
assertThat(waitForTestLatch.getCount(), is(0l));
}
// Confirm that benchmark was indeed aborted
assertThat(
future.get().state(),
isOneOf(BenchmarkResponse.State.ABORTED, BenchmarkResponse.State.COMPLETE));
}
private void validateCompetitionResult(
CompetitionResult result, BenchmarkSettings requestedSettings, boolean strict) {
// Validate settings
assertTrue(result.competitionName().startsWith(COMPETITOR_PREFIX));
assertThat(result.concurrency(), equalTo(requestedSettings.concurrency()));
assertThat(result.multiplier(), equalTo(requestedSettings.multiplier()));
// Validate node-level responses
for (CompetitionNodeResult nodeResult : result.nodeResults()) {
assertThat(nodeResult.nodeName(), notNullValue());
assertThat(nodeResult.totalIterations(), equalTo(requestedSettings.iterations()));
if (strict) {
assertThat(nodeResult.completedIterations(), equalTo(requestedSettings.iterations()));
final int expectedQueryCount =
requestedSettings.multiplier()
* nodeResult.totalIterations()
* requestedSettings.searchRequests().size();
assertThat(nodeResult.totalExecutedQueries(), equalTo(expectedQueryCount));
assertThat(nodeResult.iterations().size(), equalTo(requestedSettings.iterations()));
}
assertThat(nodeResult.warmUpTime(), greaterThanOrEqualTo(0L));
for (CompetitionIteration iteration : nodeResult.iterations()) {
// Basic sanity checks
iteration.computeStatistics();
assertThat(iteration.totalTime(), greaterThanOrEqualTo(0L));
assertThat(iteration.min(), greaterThanOrEqualTo(0L));
assertThat(iteration.max(), greaterThanOrEqualTo(iteration.min()));
assertThat(iteration.mean(), greaterThanOrEqualTo((double) iteration.min()));
assertThat(iteration.mean(), lessThanOrEqualTo((double) iteration.max()));
assertThat(iteration.queriesPerSecond(), greaterThanOrEqualTo(0.0));
assertThat(iteration.millisPerHit(), greaterThanOrEqualTo(0.0));
validatePercentiles(iteration.percentileValues());
}
}
// Validate summary statistics
final CompetitionSummary summary = result.competitionSummary();
summary.computeSummaryStatistics();
assertThat(summary, notNullValue());
assertThat(summary.getMin(), greaterThanOrEqualTo(0L));
assertThat(summary.getMax(), greaterThanOrEqualTo(summary.getMin()));
assertThat(summary.getMean(), greaterThanOrEqualTo((double) summary.getMin()));
assertThat(summary.getMean(), lessThanOrEqualTo((double) summary.getMax()));
assertThat(summary.getTotalTime(), greaterThanOrEqualTo(0L));
assertThat(summary.getQueriesPerSecond(), greaterThanOrEqualTo(0.0));
assertThat(summary.getMillisPerHit(), greaterThanOrEqualTo(0.0));
assertThat(summary.getAvgWarmupTime(), greaterThanOrEqualTo(0.0));
if (strict) {
assertThat(
(int) summary.getTotalIterations(),
equalTo(requestedSettings.iterations() * summary.nodeResults().size()));
assertThat(
(int) summary.getCompletedIterations(),
equalTo(requestedSettings.iterations() * summary.nodeResults().size()));
assertThat(
(int) summary.getTotalQueries(),
equalTo(
requestedSettings.iterations()
* requestedSettings.multiplier()
* requestedSettings.searchRequests().size()
* summary.nodeResults().size()));
validatePercentiles(summary.percentileValues);
}
}
