/** @param log Logger. */
    private void execute(GridLogger log) {
      try {
        log.info("Started execute.");

        // Countdown shared job latch so that the main thread know that all jobs are
        // inside the "execute" routine.
        jobLatch.countDown();

        log.info("After job latch.");

        // Await for the main thread to allow jobs to proceed.
        latch.await();

        log.info("After latch.");

        if (awaitMasterLeaveCallback) {
          latch0.await();

          log.info("After latch0.");
        } else log.info("Latch 0 skipped.");
      } catch (InterruptedException e) {
        // We do not expect any interruptions here, hence this statement.
        fail("Unexpected exception: " + e);
      }
    }
    /**
     * @param log Logger.
     * @param job Actual job.
     */
    private void onMasterLeave(GridLogger log, Object job) {
      log.info("Callback executed: " + job);

      latch0.countDown();

      invokeLatch.countDown();
    }
  /**
   * @param expJobs Expected jobs number.
   * @param taskStarter Task started.
   * @throws Exception If failed.
   */
  private void testMasterLeaveAwareCallback(
      int expJobs, GridClosure<GridProjection, GridFuture<?>> taskStarter) throws Exception {
    jobLatch = new CountDownLatch(expJobs);
    invokeLatch = new CountDownLatch(expJobs);

    for (int i = 0; i < GRID_CNT; i++) startGrid(i);

    int lastGridIdx = GRID_CNT - 1;

    GridFuture<?> fut = taskStarter.apply(grid(lastGridIdx).forPredicate(excludeLastPredicate()));

    jobLatch.await();

    stopGrid(lastGridIdx, true);

    latch.countDown();

    assert invokeLatch.await(5000, MILLISECONDS);

    try {
      fut.get();
    } catch (GridException e) {
      log.debug("Task failed: " + e);
    }
  }
  /**
   * Ensure that {@link GridComputeJobMasterLeaveAware} callback is invoked when fails to send
   * {@link GridJobExecuteResponse} to master node.
   *
   * @throws Exception If failed.
   */
  public void testCannotSendJobExecuteResponse() throws Exception {
    awaitMasterLeaveCallback = false;

    // Start grids.
    for (int i = 0; i < GRID_CNT; i++) startGrid(i);

    int lastGridIdx = GRID_CNT - 1;

    grid(lastGridIdx)
        .forPredicate(excludeLastPredicate())
        .compute()
        .execute(new TestTask(GRID_CNT - 1), null);

    jobLatch.await();

    for (int i = 0; i < lastGridIdx; i++)
      ((CommunicationSpi) grid(i).configuration().getCommunicationSpi()).waitLatch();

    latch.countDown();

    // Ensure that all worker nodes has already started job response sending.
    for (int i = 0; i < lastGridIdx; i++)
      ((CommunicationSpi) grid(i).configuration().getCommunicationSpi()).awaitResponse();

    // Now we stop master grid.
    stopGrid(lastGridIdx, true);

    // Release communication SPI wait latches. As master node is stopped, job worker will receive
    // and exception.
    for (int i = 0; i < lastGridIdx; i++)
      ((CommunicationSpi) grid(i).configuration().getCommunicationSpi()).releaseWaitLatch();

    assert invokeLatch.await(5000, MILLISECONDS);
  }
  /**
   * Ensure that {@link GridComputeJobMasterLeaveAware} callback is invoked on job which is
   * initiated by master and is currently running on it.
   *
   * @throws Exception If failed.
   */
  public void testLocalJobOnMaster() throws Exception {
    invokeLatch = new CountDownLatch(1);
    jobLatch = new CountDownLatch(1);

    Grid g = startGrid(0);

    g.compute().execute(new TestTask(1), null);

    jobLatch.await();

    // Count down the latch in a separate thread.
    new Thread(
            new Runnable() {
              @Override
              public void run() {
                try {
                  U.sleep(500);
                } catch (GridInterruptedException ignore) {
                  // No-op.
                }

                latch.countDown();
              }
            })
        .start();

    stopGrid(0, true);

    latch.countDown();

    assert invokeLatch.await(5000, MILLISECONDS);
  }
  /**
   * Ensure that {@link GridComputeJobMasterLeaveAware} callback is invoked when master node leaves
   * topology normally.
   *
   * @throws Exception If failed.
   */
  public void testMasterStoppedNormally() throws Exception {
    // Start grids.
    for (int i = 0; i < GRID_CNT; i++) startGrid(i);

    int lastGridIdx = GRID_CNT - 1;

    grid(lastGridIdx)
        .forPredicate(excludeLastPredicate())
        .compute()
        .execute(new TestTask(GRID_CNT - 1), null);

    jobLatch.await();

    stopGrid(lastGridIdx, true);

    latch.countDown();

    assert invokeLatch.await(5000, MILLISECONDS);
  }
  /**
   * Ensure that {@link GridComputeJobMasterLeaveAware} callback is invoked when master node leaves
   * topology abruptly (e.g. due to a network failure or immediate node shutdown).
   *
   * @throws Exception If failed.
   */
  public void testMasterStoppedAbruptly() throws Exception {
    // Start grids.
    for (int i = 0; i < GRID_CNT; i++) startGrid(i);

    int lastGridIdx = GRID_CNT - 1;

    grid(lastGridIdx)
        .forPredicate(excludeLastPredicate())
        .compute()
        .execute(new TestTask(GRID_CNT - 1), null);

    jobLatch.await();

    ((CommunicationSpi) grid(lastGridIdx).configuration().getCommunicationSpi()).blockMessages();

    stopGrid(lastGridIdx, true);

    latch.countDown();

    assert invokeLatch.await(5000, MILLISECONDS);
  }
    /**
     * Send message optionally either blocking it or throwing an exception if it is of {@link
     * GridJobExecuteResponse} type.
     *
     * @param node Destination node.
     * @param msg Message to be sent.
     * @throws GridSpiException If failed.
     */
    private void sendMessage0(GridNode node, GridTcpCommunicationMessageAdapter msg)
        throws GridSpiException {
      if (msg instanceof GridIoMessage) {
        GridIoMessage msg0 = (GridIoMessage) msg;

        if (msg0.message() instanceof GridJobExecuteResponse) {
          respLatch.countDown();

          if (wait) {
            try {
              U.await(waitLatch);
            } catch (GridInterruptedException ignore) {
              // No-op.
            }
          }
        }
      }

      if (!block) super.sendMessage(node, msg);
    }
 /** Count down wait latch. */
 private void releaseWaitLatch() {
   waitLatch.countDown();
 }