private GridProcessStatusInfo fillProcess(GridProcess process) {
WorkerStatusInfo workerInfo =
new WorkerStatusInfo(
process.getWorkerEntry().getWorkerSpecification(),
process.getHandle(),
process.getWorkerEntry().getWorkerID(),
process.getState().toString());
GridProcessStatusInfoResult result = null;
if (process.getResult() != null) {
String error = "";
String errorCause = null;
GridProcessError executionError = process.getResult().getExecutionError();
if (executionError != null) {
error = executionError.getType().getName();
if (executionError.getErrorCause() != null) {
errorCause = executionError.getErrorCause().getMessage();
}
}
result =
new GridProcessStatusInfoResult(
error,
errorCause,
process.getResult().getInitData().getElapsedTimeInMillis(),
process.getResult().getRemoteData().getElapsedTimeInMillis(),
process.getResult().getFinalData().getElapsedTimeInMillis(),
process.getResult().getExecutorResult());
SabotageCheckResult sabotageCheckResult = process.getResult().getSabotageCheckResult();
if (sabotageCheckResult != null) {
result.setSabotageCheck(sabotageCheckResult.toString());
}
}
GridProcessStatusInfo info =
new GridProcessStatusInfo(
process.getId(),
process.getTaskId(),
process.getJobId(),
process.getState().toString(),
process.getCurrentPhase().toString(),
workerInfo,
result,
process.getHandle());
info.setCreationTime(process.getCreationTime());
info.setFinalizationTime(process.getFinalizationTime());
return info;
}
private GridProcessAccounting setAccountingFields(GridProcess process) {
GridProcessAccounting accounting = process.getReplicaAccounting();
GridProcessExecutionResult result = process.getResult();
GridProcessPhasesData phasesData = new GridProcessPhasesData();
phasesData.setInitBeginning(result.getInitData().getStartTime());
phasesData.setInitEnd(result.getInitData().getEndTime());
phasesData.setRemoteBeginning(result.getRemoteData().getStartTime());
phasesData.setRemoteEnd(result.getRemoteData().getEndTime());
phasesData.setFinalBeginning(result.getFinalData().getStartTime());
phasesData.setFinalEnd(result.getFinalData().getEndTime());
phasesData.setInitOperations(result.getInitOperations());
phasesData.setGetOperations(result.getGetOperations());
accounting.setPhasesData(phasesData);
GridProcessResultInfo resultInfo = new GridProcessResultInfo();
GridProcessError error = result.getExecutionError();
if (error != null && error.getErrorCause() != null) {
resultInfo.setErrorCause(error.getErrorCause().getMessage());
resultInfo.setExecutionErrorType(error.getType().getName());
}
ExecutorResult executorResult = result.getExecutorResult();
if (executorResult != null) {
resultInfo.setExitValue(executorResult.getExitValue());
resultInfo.setStderr(executorResult.getStderr());
resultInfo.setStdout(executorResult.getStdout());
}
accounting.setResultInfo(resultInfo);
accounting.setCreationTime(process.getCreationTime());
accounting.setLatestPhase(process.getState().toString());
SabotageCheckResult sabotageCheckResult = result.getSabotageCheckResult();
String sabotageCheck = sabotageCheckResult == null ? null : sabotageCheckResult.toString();
accounting.setSabotageCheck(sabotageCheck);
accounting.setTaskSequenceNumber(process.getSpec().getTaskSequenceNumber());
accounting.setGridProcessSequenceNumber(process.getId());
accounting.setState(process.getState());
return accounting;
}
private void abortReplicaSisters(GridProcess execution, List<IResponseTO> responses) {
for (GridProcess sisterGridProcess : execution.getTask().getGridProcesses()) {
if (!sisterGridProcess.equals(execution) && sisterGridProcess.getState().isRunnable()) {
abort(sisterGridProcess, responses);
}
}
}
private boolean hasAnySisterGridProcessFinished(GridProcess execution) {
for (GridProcess sisterGridProcess : execution.getTask().getGridProcesses()) {
if (!sisterGridProcess.equals(execution)
&& sisterGridProcess.getState().equals(GridProcessState.FINISHED)) {
return true;
}
}
return false;
}
private void abort(GridProcess gridProcess, List<IResponseTO> responses) {
if (gridProcess.getState().isRunnable()) {
gridProcess.setGridProcessState(GridProcessState.ABORTED);
gridProcess.getReplicaAccounting().setState(GridProcessState.ABORTED);
GridProcessAccounting accounting = setAccountingFields(gridProcess);
accounting.setTransfersProgress(convertTransfer(gridProcess.getTransfersProgress()));
reportReplicaAccounting(gridProcess, responses);
gridProcess.getOperations().cancelOperations(responses);
executionEnded(gridProcess, GridProcessState.ABORTED, responses);
}
}
private boolean verifyRunningProccess(Task task) {
int running = 0;
for (GridProcess replica : task.getGridProcesses()) {
GridProcessState replicaState = replica.getState();
if (GridProcessState.RUNNING.equals(replicaState)
|| GridProcessState.UNSTARTED.equals(replicaState)) {
running++;
}
}
return running < maxReplicas
&& (GridProcessState.RUNNING.equals(task.getState())
|| GridProcessState.UNSTARTED.equals(task.getState()));
}
public void executionFailed(GridProcess execution, List<IResponseTO> responses) {
reportReplicaAccounting(execution, responses);
Job job = execution.getJob();
GridProcessExecutionResult executionResult = execution.getResult();
try {
job.newReplicaResult(
executionResult,
GridProcessState.FAILED,
verifyFailure(execution.getTask(), GridProcessState.FAILED),
canReplicate(execution.getTask()));
} catch (IllegalResultException e) {
responses.add(
new LoggerResponseTO(
"Illegal result on replica " + execution.getState() + " : " + e.getMessage(),
LoggerResponseTO.ERROR));
}
GridProcessHandle handle = executionResult.getReplicaHandle();
WorkerEntry workerEntry = execution.getWorkerEntry();
workerEntry.deallocate();
GridProcessErrorTypes type = null;
if (executionResult != null && executionResult.getExecutionError() != null) {
type = executionResult.getExecutionError().getType();
}
boolean enteredTaskBlacklist = executionFailedOnWorker(workerEntry, type, execution, responses);
if (enteredTaskBlacklist) {
if (!isWorkerNeeded(workerEntry, execution)) {
unwantWorker(job, workerEntry, responses);
}
} else {
disposeWorker(workerEntry, responses);
}
boolean hasJobEnded = hasJobEnded(job);
String executorMsg = "";
if (executionResult != null
&& executionResult.getExecutionError() != null
&& executionResult.getExecutionError().getErrorCause() != null) {
executorMsg = executionResult.getExecutionError().getErrorCause().toString();
}
responses.add(
new LoggerResponseTO(
"Grid process "
+ execution.getState()
+ " "
+ handle
+ ". Job ended: "
+ hasJobEnded
+ " "
+ executorMsg
+ ".",
LoggerResponseTO.DEBUG));
if (hasJobEnded) {
finishJob(execution.getJob(), responses);
}
if (!isJobSatisfied(job) && !hasJobEnded) {
Request request = execution.getJob().getRequest(workerEntry.getRequestID());
if (request != null) {
request.setPaused(false);
}
ResumeRequestResponseTO to = new ResumeRequestResponseTO();
to.setPeerAddress(StringUtil.deploymentIDToAddress(workerEntry.getPeerID()));
to.setRequestID(workerEntry.getRequestID());
responses.add(to);
}
updateScheduler(responses);
}
public JobWorkerStatus getCompleteStatus() {
Map<Integer, Job> jobsMap = JobInfo.getInstance().getJobs();
Map<Integer, Set<WorkerEntry>> workersByJob = CommonUtils.createMap();
JobStatusInfo jobInfo = null;
List<TaskStatusInfo> tasksList = null;
Map<Integer, JobStatusInfo> jobs = CommonUtils.createSerializableMap();
// Jobs
for (Job job : jobsMap.values()) {
Set<WorkerEntry> workers = new LinkedHashSet<WorkerEntry>();
tasksList = new ArrayList<TaskStatusInfo>();
for (Task task : job.getTasks()) {
tasksList.add(fillTask(task));
for (GridProcess gridProcess : task.getGridProcesses()) {
if (gridProcess.getState() == GridProcessState.RUNNING)
workers.add(gridProcess.getWorkerEntry());
}
}
jobInfo =
new JobStatusInfo(
job.getJobId(),
job.getSpec(),
UtilConverter.getJobState(job.getState()),
tasksList,
job.getCreationTime(),
job.getFinalizationTime());
jobs.put(jobInfo.getJobId(), jobInfo);
if (job.isRunning()) {
workersByJob.put(job.getJobId(), workers);
}
}
Map<Integer, WorkerStatusInfo[]> workers = CommonUtils.createSerializableMap();
WorkerStatusInfo[] workerList = null;
for (Entry<Integer, Set<WorkerEntry>> entry : workersByJob.entrySet()) {
workerList = workers.get(entry.getKey());
if (workerList == null) {
workerList = new WorkerStatusInfo[entry.getValue().size()];
workers.put(entry.getKey(), workerList);
}
int i = 0;
for (WorkerEntry workerEntry : entry.getValue()) {
GridProcessHandle handle = null;
String state = null;
if (workerEntry.getGridProcess() != null) {
handle = workerEntry.getGridProcess().getHandle();
state = workerEntry.getGridProcess().getState().toString();
}
workerList[i] =
new WorkerStatusInfo(
workerEntry.getWorkerSpecification(), handle, workerEntry.getWorkerID(), state);
i++;
}
}
JobWorkerStatus status = new JobWorkerStatus(jobs, workers);
return status;
}