/**
* @param log Logger.
* @param job Actual job.
*/
private void onMasterLeave(GridLogger log, Object job) {
log.info("Callback executed: " + job);
latch0.countDown();
invokeLatch.countDown();
}
/** @param log Logger. */
private void execute(GridLogger log) {
try {
log.info("Started execute.");
// Countdown shared job latch so that the main thread know that all jobs are
// inside the "execute" routine.
jobLatch.countDown();
log.info("After job latch.");
// Await for the main thread to allow jobs to proceed.
latch.await();
log.info("After latch.");
if (awaitMasterLeaveCallback) {
latch0.await();
log.info("After latch0.");
} else log.info("Latch 0 skipped.");
} catch (InterruptedException e) {
// We do not expect any interruptions here, hence this statement.
fail("Unexpected exception: " + e);
}
}
/**
* @param expJobs Expected jobs number.
* @param taskStarter Task started.
* @throws Exception If failed.
*/
private void testMasterLeaveAwareCallback(
int expJobs, GridClosure<GridProjection, GridFuture<?>> taskStarter) throws Exception {
jobLatch = new CountDownLatch(expJobs);
invokeLatch = new CountDownLatch(expJobs);
for (int i = 0; i < GRID_CNT; i++) startGrid(i);
int lastGridIdx = GRID_CNT - 1;
GridFuture<?> fut = taskStarter.apply(grid(lastGridIdx).forPredicate(excludeLastPredicate()));
jobLatch.await();
stopGrid(lastGridIdx, true);
latch.countDown();
assert invokeLatch.await(5000, MILLISECONDS);
try {
fut.get();
} catch (GridException e) {
log.debug("Task failed: " + e);
}
}
/**
* Ensure that {@link GridComputeJobMasterLeaveAware} callback is invoked when fails to send
* {@link GridJobExecuteResponse} to master node.
*
* @throws Exception If failed.
*/
public void testCannotSendJobExecuteResponse() throws Exception {
awaitMasterLeaveCallback = false;
// Start grids.
for (int i = 0; i < GRID_CNT; i++) startGrid(i);
int lastGridIdx = GRID_CNT - 1;
grid(lastGridIdx)
.forPredicate(excludeLastPredicate())
.compute()
.execute(new TestTask(GRID_CNT - 1), null);
jobLatch.await();
for (int i = 0; i < lastGridIdx; i++)
((CommunicationSpi) grid(i).configuration().getCommunicationSpi()).waitLatch();
latch.countDown();
// Ensure that all worker nodes has already started job response sending.
for (int i = 0; i < lastGridIdx; i++)
((CommunicationSpi) grid(i).configuration().getCommunicationSpi()).awaitResponse();
// Now we stop master grid.
stopGrid(lastGridIdx, true);
// Release communication SPI wait latches. As master node is stopped, job worker will receive
// and exception.
for (int i = 0; i < lastGridIdx; i++)
((CommunicationSpi) grid(i).configuration().getCommunicationSpi()).releaseWaitLatch();
assert invokeLatch.await(5000, MILLISECONDS);
}
/**
* Ensure that {@link GridComputeJobMasterLeaveAware} callback is invoked on job which is
* initiated by master and is currently running on it.
*
* @throws Exception If failed.
*/
public void testLocalJobOnMaster() throws Exception {
invokeLatch = new CountDownLatch(1);
jobLatch = new CountDownLatch(1);
Grid g = startGrid(0);
g.compute().execute(new TestTask(1), null);
jobLatch.await();
// Count down the latch in a separate thread.
new Thread(
new Runnable() {
@Override
public void run() {
try {
U.sleep(500);
} catch (GridInterruptedException ignore) {
// No-op.
}
latch.countDown();
}
})
.start();
stopGrid(0, true);
latch.countDown();
assert invokeLatch.await(5000, MILLISECONDS);
}
/**
* Ensure that {@link GridComputeJobMasterLeaveAware} callback is invoked when master node leaves
* topology normally.
*
* @throws Exception If failed.
*/
public void testMasterStoppedNormally() throws Exception {
// Start grids.
for (int i = 0; i < GRID_CNT; i++) startGrid(i);
int lastGridIdx = GRID_CNT - 1;
grid(lastGridIdx)
.forPredicate(excludeLastPredicate())
.compute()
.execute(new TestTask(GRID_CNT - 1), null);
jobLatch.await();
stopGrid(lastGridIdx, true);
latch.countDown();
assert invokeLatch.await(5000, MILLISECONDS);
}
/**
* Send message optionally either blocking it or throwing an exception if it is of {@link
* GridJobExecuteResponse} type.
*
* @param node Destination node.
* @param msg Message to be sent.
* @throws GridSpiException If failed.
*/
private void sendMessage0(GridNode node, GridTcpCommunicationMessageAdapter msg)
throws GridSpiException {
if (msg instanceof GridIoMessage) {
GridIoMessage msg0 = (GridIoMessage) msg;
if (msg0.message() instanceof GridJobExecuteResponse) {
respLatch.countDown();
if (wait) {
try {
U.await(waitLatch);
} catch (GridInterruptedException ignore) {
// No-op.
}
}
}
}
if (!block) super.sendMessage(node, msg);
}
/**
* Ensure that {@link GridComputeJobMasterLeaveAware} callback is invoked when master node leaves
* topology abruptly (e.g. due to a network failure or immediate node shutdown).
*
* @throws Exception If failed.
*/
public void testMasterStoppedAbruptly() throws Exception {
// Start grids.
for (int i = 0; i < GRID_CNT; i++) startGrid(i);
int lastGridIdx = GRID_CNT - 1;
grid(lastGridIdx)
.forPredicate(excludeLastPredicate())
.compute()
.execute(new TestTask(GRID_CNT - 1), null);
jobLatch.await();
((CommunicationSpi) grid(lastGridIdx).configuration().getCommunicationSpi()).blockMessages();
stopGrid(lastGridIdx, true);
latch.countDown();
assert invokeLatch.await(5000, MILLISECONDS);
}
/** Count down wait latch. */
private void releaseWaitLatch() {
waitLatch.countDown();
}