private void validateAllocatedContainers(boolean isReplicated, String containerTemplateLink)
        throws Throwable {
      QueryTask.Query kindClause =
          new QueryTask.Query()
              .setTermPropertyName(ServiceDocument.FIELD_NAME_KIND)
              .setTermMatchValue(Utils.buildKind(ContainerService.State.class));

      QueryTask.Query containerTemplateClause =
          new QueryTask.Query()
              .setTermPropertyName(
                  ContainerService.State.FIELD_NAME_CONTAINER_TEMPLATE_SERVICE_LINK)
              .setTermMatchValue(containerTemplateLink);

      QueryTask.QuerySpecification querySpecification = new QueryTask.QuerySpecification();
      querySpecification.query.addBooleanClause(kindClause);
      querySpecification.query.addBooleanClause(containerTemplateClause);

      QueryTask query = QueryTask.create(querySpecification).setDirect(true);

      NodeGroupBroadcastResponse queryResponse = testEnvironment.sendBroadcastQueryAndWait(query);
      Set<String> documentLinks = QueryTaskUtils.getBroadcastQueryResults(queryResponse);

      // Verify that count(replicas) == count(dockerVms) i.e. 1 container per vm
      int expectedReplicaCount = isReplicated ? dockerVms.size() : 1;
      assertThat(documentLinks.size(), is(expectedReplicaCount));

      // Verify that each container was assigned to a unique docker vm
      Set<String> uniqueVmLinks = new HashSet<>();
      for (String documentLink : documentLinks) {
        ContainerService.State state =
            testEnvironment.getServiceState(documentLink, ContainerService.State.class);
        uniqueVmLinks.add(state.vmServiceLink);
      }
      assertThat(uniqueVmLinks.size(), is(expectedReplicaCount));
    }
  /**
   * Triggers a query to retrieve the "child" ImageCopyService instances in FAILED or CANCELLED
   * state.
   *
   * @param current
   */
  private void checkFailedOrCancelledCount(final State current) {
    Operation.CompletionHandler handler =
        new Operation.CompletionHandler() {
          @Override
          public void handle(Operation completedOp, Throwable failure) {
            if (failure != null) {
              // The query failed to execute. This most likely means that the
              // host is in a bad state and if we re-issue the query it is likely
              // to fail again. Terminate and fail the task early and delegate any
              // retry logic to the caller.
              failTask(failure);
              return;
            }

            QueryTask rsp = completedOp.getBody(QueryTask.class);

            State s = buildPatch(current.taskInfo.stage, current.taskInfo.subStage, null);
            ServiceUtils.logInfo(
                ImageReplicatorService.this, "Failed %s", Utils.toJson(rsp.results.documentLinks));
            s.failedOrCanceledCopies = rsp.results.documentLinks.size();

            sendSelfPatch(s);
          }
        };

    QueryTask.QuerySpecification spec =
        QueryTaskUtils.buildChildServiceTaskStatusQuerySpec(
            this.getSelfLink(),
            ImageCopyService.State.class,
            TaskState.TaskStage.FAILED,
            TaskState.TaskStage.CANCELLED);

    this.sendQuery(spec, handler);
  }
  /**
   * This method queries for the document link of the cluster configuration for the Kubernetes
   * Cluster.
   *
   * @param currentState
   */
  private void queryClusterConfiguration(final KubernetesClusterCreateTask currentState) {
    QueryTask.Query kindClause =
        new QueryTask.Query()
            .setTermPropertyName(ServiceDocument.FIELD_NAME_KIND)
            .setTermMatchValue(Utils.buildKind(ClusterConfigurationService.State.class));

    QueryTask.Query idClause =
        new QueryTask.Query()
            .setTermPropertyName(ClusterConfigurationService.State.FIELD_NAME_SELF_LINK)
            .setTermMatchValue(
                ClusterConfigurationServiceFactory.SELF_LINK
                    + "/"
                    + ClusterType.KUBERNETES.toString().toLowerCase());

    QueryTask.QuerySpecification querySpecification = new QueryTask.QuerySpecification();
    querySpecification.query.addBooleanClause(kindClause);
    querySpecification.query.addBooleanClause(idClause);
    QueryTask queryTask = QueryTask.create(querySpecification).setDirect(true);

    sendRequest(
        HostUtils.getCloudStoreHelper(this)
            .createBroadcastPost(
                ServiceUriPaths.CORE_LOCAL_QUERY_TASKS, ServiceUriPaths.DEFAULT_NODE_SELECTOR)
            .setBody(queryTask)
            .setCompletion(
                (Operation operation, Throwable throwable) -> {
                  if (null != throwable) {
                    failTask(throwable);
                    return;
                  }

                  NodeGroupBroadcastResponse queryResponse =
                      operation.getBody(NodeGroupBroadcastResponse.class);
                  Set<String> documentLinks =
                      QueryTaskUtils.getBroadcastQueryResults(queryResponse);
                  if (documentLinks.isEmpty()) {
                    failTask(
                        new IllegalStateException(
                            String.format(
                                "Cannot find cluster configuration for %s",
                                ClusterType.KUBERNETES.toString())));
                    return;
                  }

                  retrieveClusterConfiguration(currentState, documentLinks.iterator().next());
                }));
  }