@Test
  public void testSubmitBenchmark() throws Exception {
    final int iters =
        between(1, 3); // we run this more than once to make sure metadata is cleaned up propperly
    for (int i = 0; i < iters; i++) {
      final BenchmarkRequest request =
          BenchmarkTestUtil.randomRequest(
              client(), indices, numExecutorNodes, competitionSettingsMap);
      logger.info(
          "--> Submitting benchmark - competitors [{}] iterations [{}]",
          request.competitors().size(),
          request.settings().iterations());
      final BenchmarkResponse response = client().bench(request).actionGet();

      assertThat(response, notNullValue());
      assertThat(response.state(), equalTo(BenchmarkResponse.State.COMPLETE));
      assertFalse(response.hasErrors());
      assertThat(response.benchmarkName(), equalTo(BENCHMARK_NAME));
      assertThat(response.competitionResults().size(), equalTo(request.competitors().size()));

      for (CompetitionResult result : response.competitionResults().values()) {
        assertThat(result.nodeResults().size(), equalTo(numExecutorNodes));
        validateCompetitionResult(
            result, competitionSettingsMap.get(result.competitionName()), true);
      }
    }
  }
  @Test
  public void testBenchmarkWithErrors() {
    List<SearchRequest> reqList = new ArrayList<>();
    int numQueries = scaledRandomIntBetween(20, 100);
    int numErrors = scaledRandomIntBetween(1, numQueries);
    final boolean containsFatal = randomBoolean();
    if (containsFatal) {
      ScriptScoreFunctionBuilder scriptFunction =
          scriptFunction("DOES NOT COMPILE - fails on any shard");
      SearchRequest searchRequest =
          searchRequest()
              .source(
                  searchSource()
                      .query(functionScoreQuery(FilterBuilders.matchAllFilter(), scriptFunction)));
      reqList.add(searchRequest);
    }
    for (int i = 0; reqList.size() < numErrors; i++) {
      ScriptScoreFunctionBuilder scriptFunction = scriptFunction("throw new RuntimeException();");
      SearchRequest searchRequest =
          searchRequest()
              .source(
                  searchSource()
                      .query(functionScoreQuery(FilterBuilders.matchAllFilter(), scriptFunction)));
      reqList.add(searchRequest);
    }
    logger.info("--> run with [{}] errors ", numErrors);
    for (int i = 0; reqList.size() < numQueries; i++) {

      reqList.add(BenchmarkTestUtil.randomSearch(client(), indices));
    }
    Collections.shuffle(reqList, getRandom());

    final BenchmarkRequest request =
        BenchmarkTestUtil.randomRequest(
            client(),
            indices,
            numExecutorNodes,
            competitionSettingsMap,
            reqList.toArray(new SearchRequest[0]));
    logger.info(
        "--> Submitting benchmark - competitors [{}] iterations [{}]",
        request.competitors().size(),
        request.settings().iterations());
    final BenchmarkResponse response = client().bench(request).actionGet();

    assertThat(response, notNullValue());
    if (response.hasErrors() || containsFatal) {
      assertThat(response.state(), equalTo(BenchmarkResponse.State.FAILED));
    } else {
      assertThat(response.state(), equalTo(BenchmarkResponse.State.COMPLETE));
      for (CompetitionResult result : response.competitionResults().values()) {
        assertThat(result.nodeResults().size(), equalTo(numExecutorNodes));
        validateCompetitionResult(
            result, competitionSettingsMap.get(result.competitionName()), true);
      }
    }
    assertThat(response.benchmarkName(), equalTo(BENCHMARK_NAME));
  }
  @Test
  public void testAbortBenchmark() throws Exception {
    final int iters =
        between(1, 3); // we run this more than once to make sure metadata is cleaned up propperly
    for (int i = 0; i < iters; i++) {
      SearchRequest searchRequest = prepareBlockingScriptQuery();
      final BenchmarkRequest request =
          BenchmarkTestUtil.randomRequest(
              client(), indices, numExecutorNodes, competitionSettingsMap, searchRequest);
      request.settings().iterations(Integer.MAX_VALUE, true); // massive amount of iterations
      logger.info(
          "--> Submitting benchmark - competitors [{}] iterations [{}]",
          request.competitors().size(),
          request.settings().iterations());
      boolean aborted = false;
      final ActionFuture<BenchmarkResponse> benchmarkResponse = client().bench(request);
      try {
        waitForQuery.await();
        final AbortBenchmarkResponse abortResponse =
            client().prepareAbortBench(BENCHMARK_NAME).get();
        aborted = true;
        // Confirm that the benchmark was actually aborted and did not finish on its own
        assertAcked(abortResponse);
        // Confirm that there are no active benchmarks in the cluster
        final BenchmarkStatusResponse statusResponse =
            client().prepareBenchStatus().execute().actionGet();
        waitForTestLatch.countDown(); // let the queries go - we already aborted and got the status
        assertThat(statusResponse.totalActiveBenchmarks(), equalTo(0));

        // Confirm that benchmark was indeed aborted
        assertThat(benchmarkResponse.get().state(), is(BenchmarkResponse.State.ABORTED));

      } finally {
        if (waitForTestLatch.getCount() == 1) {
          waitForTestLatch.countDown();
        }
        if (!aborted) {
          client().prepareAbortBench(BENCHMARK_NAME).get();
        }
        assertThat(waitForTestLatch.getCount(), is(0l));
      }
    }
  }
  @Test
  public void testListBenchmarks() throws Exception {
    SearchRequest searchRequest = prepareBlockingScriptQuery();
    final BenchmarkRequest request =
        BenchmarkTestUtil.randomRequest(
            client(), indices, numExecutorNodes, competitionSettingsMap, searchRequest);
    logger.info(
        "--> Submitting benchmark - competitors [{}] iterations [{}]",
        request.competitors().size(),
        request.settings().iterations());

    final ActionFuture<BenchmarkResponse> future = client().bench(request);
    try {
      waitForQuery.await();
      final BenchmarkStatusResponse statusResponse =
          client().prepareBenchStatus().execute().actionGet();
      waitForTestLatch.countDown();
      assertThat(statusResponse.benchmarkResponses().size(), equalTo(1));
      for (BenchmarkResponse benchmarkResponse : statusResponse.benchmarkResponses()) {
        assertThat(benchmarkResponse.benchmarkName(), equalTo(BENCHMARK_NAME));
        assertThat(benchmarkResponse.state(), equalTo(BenchmarkResponse.State.RUNNING));
        assertFalse(benchmarkResponse.hasErrors());

        for (CompetitionResult result : benchmarkResponse.competitionResults().values()) {
          assertThat(result.nodeResults().size(), lessThanOrEqualTo(numExecutorNodes));
          validateCompetitionResult(
              result, competitionSettingsMap.get(result.competitionName()), false);
        }
      }

    } finally {
      if (waitForTestLatch.getCount() == 1) {
        waitForTestLatch.countDown();
      }
      client().prepareAbortBench(BENCHMARK_NAME).get();
      // Confirm that there are no active benchmarks in the cluster
      assertThat(
          client().prepareBenchStatus().execute().actionGet().totalActiveBenchmarks(), equalTo(0));
      assertThat(waitForTestLatch.getCount(), is(0l));
    }
    // Confirm that benchmark was indeed aborted
    assertThat(
        future.get().state(),
        isOneOf(BenchmarkResponse.State.ABORTED, BenchmarkResponse.State.COMPLETE));
  }
  @Test
  public void testAbortByPattern() throws Exception {
    final int iters =
        between(1, 3); // we run this more than once to make sure metadata is cleaned up propperly
    for (int i = 0; i < iters; i++) {
      List<BenchmarkRequest> requests = new ArrayList<>();
      List<ActionFuture<BenchmarkResponse>> responses = new ArrayList<>();

      SearchRequest searchRequest = prepareBlockingScriptQuery();
      final int benches = between(1, 3);
      String[] names = new String[benches];
      for (int k = 0; k < benches; k++) {
        final BenchmarkRequest request =
            BenchmarkTestUtil.randomRequest(
                client(), indices, numExecutorNodes, competitionSettingsMap, searchRequest);
        request.settings().iterations(Integer.MAX_VALUE, true); // massive amount of iterations
        names[k] = BENCHMARK_NAME + Integer.toString(k);
        request.benchmarkName(names[k]);
        requests.add(request);
        logger.info(
            "--> Submitting benchmark - competitors [{}] iterations [{}]",
            request.competitors().size(),
            request.settings().iterations());
      }

      boolean aborted = false;
      for (BenchmarkRequest r : requests) {
        final ActionFuture<BenchmarkResponse> benchmarkResponse = client().bench(r);
        responses.add(benchmarkResponse);
      }
      try {
        waitForQuery.await();
        if (benches > 1) {
          awaitBusy(
              new Predicate<Object>() {
                @Override
                public boolean apply(java.lang.Object input) {
                  return client().prepareBenchStatus().get().benchmarkResponses().size() == benches;
                }
              });
        }
        final String badPatternA = "*z";
        final String badPatternB = "xxx";
        final String[] patterns;
        switch (getRandom().nextInt(3)) {
          case 0:
            patterns = new String[] {"*"};
            break;
          case 1:
            patterns = new String[] {BENCHMARK_NAME_WILDCARD, badPatternA, badPatternB};
            break;
          case 2:
            patterns = names;
            break;
          default:
            patterns = new String[] {BENCHMARK_NAME_WILDCARD};
        }
        final AbortBenchmarkResponse abortResponse = client().prepareAbortBench(patterns).get();
        aborted = true;
        assertAcked(abortResponse);

        // Confirm that there are no active benchmarks in the cluster
        final BenchmarkStatusResponse statusResponse =
            client().prepareBenchStatus().execute().actionGet();
        waitForTestLatch.countDown(); // let the queries go - we already aborted and got the status
        assertThat(statusResponse.totalActiveBenchmarks(), equalTo(0));

        // Confirm that benchmark was indeed aborted
        for (ActionFuture<BenchmarkResponse> r : responses) {
          assertThat(r.get().state(), is(BenchmarkResponse.State.ABORTED));
        }
      } finally {
        if (waitForTestLatch.getCount() == 1) {
          waitForTestLatch.countDown();
        }
        if (!aborted) {
          client().prepareAbortBench(BENCHMARK_NAME).get();
        }
        assertThat(waitForTestLatch.getCount(), is(0l));
      }
    }
  }