/**
* Creates new shuffle job.
*
* @param jobId Job ID.
* @return Created shuffle job.
* @throws GridException If job creation failed.
*/
private GridHadoopShuffleJob<UUID> newJob(GridHadoopJobId jobId) throws GridException {
GridHadoopMapReducePlan plan = ctx.jobTracker().plan(jobId);
GridHadoopShuffleJob<UUID> job =
new GridHadoopShuffleJob<>(
ctx.localNodeId(),
ctx.localNodeId(),
log,
ctx.jobTracker().job(jobId, null),
mem,
plan.reducers(),
plan.reducers(ctx.localNodeId()));
UUID[] rdcAddrs = new UUID[plan.reducers()];
for (int i = 0; i < rdcAddrs.length; i++) {
UUID nodeId = plan.nodeForReducer(i);
assert nodeId != null : "Plan is missing node for reducer [plan=" + plan + ", rdc=" + i + ']';
rdcAddrs[i] = nodeId;
}
boolean init = job.initializeReduceAddresses(rdcAddrs);
assert init;
return job;
}
/**
* Flushes all the outputs for the given job to remote nodes.
*
* @param jobId Job ID.
* @return Future.
*/
public GridFuture<?> flush(GridHadoopJobId jobId) {
GridHadoopShuffleJob job = jobs.get(jobId);
if (job == null) return new GridFinishedFutureEx<>();
try {
return job.flush();
} catch (GridException e) {
return new GridFinishedFutureEx<>(e);
}
}
/** @param jobId Job id. */
public void jobFinished(GridHadoopJobId jobId) {
GridHadoopShuffleJob job = jobs.remove(jobId);
if (job != null) {
try {
job.close();
} catch (GridException e) {
U.error(log, "Failed to close job: " + jobId, e);
}
}
}
/**
* Stops shuffle.
*
* @param cancel If should cancel all ongoing activities.
*/
@Override
public void stop(boolean cancel) {
for (GridHadoopShuffleJob job : jobs.values()) {
try {
job.close();
} catch (GridException e) {
U.error(log, "Failed to close job.", e);
}
}
jobs.clear();
}
/**
* @param jobId Task info.
* @return Shuffle job.
*/
private GridHadoopShuffleJob<UUID> job(GridHadoopJobId jobId) throws GridException {
GridHadoopShuffleJob<UUID> res = jobs.get(jobId);
if (res == null) {
res = newJob(jobId);
GridHadoopShuffleJob<UUID> old = jobs.putIfAbsent(jobId, res);
if (old != null) {
res.close();
res = old;
} else if (res.reducersInitialized()) startSending(res);
}
return res;
}
/**
* Starts message sending thread.
*
* @param shuffleJob Job to start sending for.
*/
private void startSending(GridHadoopShuffleJob<UUID> shuffleJob) {
shuffleJob.startSending(
ctx.kernalContext().gridName(),
new GridInClosure2X<UUID, GridHadoopShuffleMessage>() {
@Override
public void applyx(UUID dest, GridHadoopShuffleMessage msg) throws GridException {
send0(dest, msg);
}
});
}