/**
* Process message from the grid engine itself. In case the process failed, we keep the work
* packet around so the developer can reproduce the error.
*
* @param w Work packet whose state changed
*/
@Override
public void stateChanged(final GridWorkPacket w) {
if (w == null) {
return;
}
// We report state change just once.
if (!reported) {
try {
if (w.getPassed()) {
// This is the last response we will send - request is completed.
// There might have been an error from RMI, check that
if (allocatorListener.getLastThrowable() == null) {
sendResponse(
request, new DaemonProgressMessage(DaemonProgress.RequestCompleted), true);
} else {
sendResponse(
request, new DaemonException(allocatorListener.getLastThrowable()), true);
}
} else if (w.getFailed()) {
// This is the last response we will send - request failed
if (allocatorListener.getLastThrowable() == null) {
sendResponse(request, new DaemonException(w.getErrorMessage()), true);
} else {
sendResponse(
request,
new DaemonException(w.getErrorMessage(), allocatorListener.getLastThrowable()),
true);
}
}
reported = true;
} finally {
if (!w.getFailed()) {
// Delete workPacket file
LOGGER.debug("Deleting sge packet file: " + sgePacketFile.getAbsolutePath());
FileUtilities.quietDelete(sgePacketFile);
} else {
LOGGER.warn("Retaining sge packet file: " + sgePacketFile.getAbsolutePath());
}
}
}
}
/** Process failure, return more descriptive exception */
private DaemonException processFailedJob(
final GridWorkPacket gridWorkPacket, final File packageFile, final Exception exception) {
final DaemonException daemonException;
final File storedFile = failedJobManager.storeFile(packageFile);
if (storedFile != null) {
daemonException =
new DaemonException(
MessageFormat.format(
"Failed passing work packet to grid engine:\n{0}\nUse {1} for the --sge parameter",
gridWorkPacket.toString(), storedFile.getAbsolutePath()),
exception);
} else {
daemonException =
new DaemonException(
"Failed passing work packet to grid engine:\n" + gridWorkPacket.toString(),
exception);
}
FileUtilities.quietDelete(packageFile);
LOGGER.error(MprcException.getDetailedMessage(daemonException), daemonException);
return daemonException;
}
@Override
protected void processRequest(final DaemonRequest request) {
final GridWorkPacket gridWorkPacket =
getBaseGridWorkPacket(gridScriptFactory.getApplicationName(wrapperScript));
final File sgePacketFile =
new File(getSharedTempDirectory(), queueName + "_" + uniqueId.incrementAndGet());
try {
final SgeMessageListener allocatorListener = new SgeMessageListener(request);
final SgePacket sgePacket =
new SgePacket(
serviceFactory.serializeRequest(
request.getWorkPacket(), getDaemon().getResponseDispatcher(), allocatorListener),
daemonConnection.getConnectionName(),
fileTokenFactory.getDaemonConfigInfo(),
getDaemonLoggerFactory().getLogFolder());
writeWorkerAllocatorInputObject(sgePacketFile, sgePacket);
final List<String> parameters = gridScriptFactory.getParameters(wrapperScript, sgePacketFile);
gridWorkPacket.setParameters(parameters);
// Set our own listener to the work packet progress. When the packet returns, the execution
// will be resumed
final MyWorkPacketStateListener listener =
new MyWorkPacketStateListener(request, sgePacketFile, allocatorListener);
gridWorkPacket.setListener(listener);
gridWorkPacket.setPriority(request.getWorkPacket().getPriority());
// Run the job
final String requestId = manager.passToGridEngine(gridWorkPacket);
// Report the information about the running task to the caller, making sure they get the task
// id and the logs
final AssignedTaskData data = new AssignedTaskData(requestId);
final RunnerProgressReporter reporter = new RunnerProgressReporter(this, request);
// Report the assigned ID
reporter.reportProgress(data);
final ParentLog log =
getDaemonLoggerFactory().createLog(request.getWorkPacket().getTaskId(), reporter);
// Report that we spawned a child with its own SGE log, we use the SGE-based log paths for
// this
log.createChildLog(
gridWorkPacket.getOutputLogFilePath(), gridWorkPacket.getErrorLogFilePath());
// We are not done yet! The grid work packet's progress listener will get called when the
// state of the task changes,
// and either mark the task failed or successful.
} catch (Exception t) {
final DaemonException daemonException = processFailedJob(gridWorkPacket, sgePacketFile, t);
sendResponse(request, daemonException, true);
throw daemonException;
}
}
private GridWorkPacket getBaseGridWorkPacket(final String command) {
final GridWorkPacket gridWorkPacket = new GridWorkPacket(command, null);
gridWorkPacket.setNativeSpecification(nativeSpecification);
gridWorkPacket.setQueueName(queueName);
gridWorkPacket.setMemoryRequirement(memoryRequirement);
gridWorkPacket.setWorkingFolder(new File(".").getAbsolutePath());
gridWorkPacket.setLogFolder(
FileUtilities.getDateBasedDirectory(getDaemonLoggerFactory().getLogFolder(), new Date())
.getAbsolutePath());
return gridWorkPacket;
}
