public void disposeWorker(WorkerEntry workerEntry, List<IResponseTO> responses) {
ServiceID serviceID = workerEntry.getServiceID();
responses.add(new LoggerResponseTO("Worker dispose: " + serviceID, LoggerResponseTO.DEBUG));
workerEntry.dispose();
WorkerDAO workerDAO = BrokerDAOFactory.getInstance().getWorkerDAO();
String wAddress = StringUtil.deploymentIDToAddress(workerEntry.getWorkerID());
DisposeWorkerResponseTO disposeTO = new DisposeWorkerResponseTO();
disposeTO.setPeerAddress(StringUtil.deploymentIDToAddress(workerEntry.getPeerID()));
disposeTO.setWorkerAddress(wAddress);
disposeTO.setWorkerPublicKey(workerDAO.getWorkerPublicKey(wAddress));
responses.add(disposeTO);
responses.add(new LoggerResponseTO("Stub to be released: " + wAddress, LoggerResponseTO.INFO));
ReleaseResponseTO releaseTO = new ReleaseResponseTO();
releaseTO.setStubAddress(wAddress);
responses.add(releaseTO);
workerDAO.removeWorker(wAddress);
WorkerInfo.getInstance().removeWorker(workerEntry.getServiceID().getContainerID().toString());
}
/**
* @param job
* @param workerEntry
*/
private void unwantWorker(Job job, WorkerEntry workerEntry, List<IResponseTO> responses) {
responses.add(
new LoggerResponseTO(
"Worker unwanted: " + workerEntry.getWorkerID(), LoggerResponseTO.DEBUG));
job.unwantWorker(workerEntry);
workerEntry.unwant();
RequestSpecification spec = workerEntry.getRequestSpecification();
String peerAddress = StringUtil.deploymentIDToAddress(workerEntry.getPeerID());
String workerAddress = StringUtil.deploymentIDToAddress(workerEntry.getWorkerID());
Integer jobId = Integer.parseInt("" + spec.getJobId());
JobSpecification jobSpec = BrokerDAOFactory.getInstance().getJobDAO().getJobSpec(jobId);
UnwantWorkerResponseTO unwantWorkerTO = new UnwantWorkerResponseTO();
unwantWorkerTO.setJobID(jobId);
unwantWorkerTO.setJobSpec(jobSpec);
unwantWorkerTO.setMaxFails(spec.getMaxFails());
unwantWorkerTO.setMaxReplicas(spec.getMaxReplicas());
unwantWorkerTO.setPeerAddress(peerAddress);
unwantWorkerTO.setRequestID(spec.getRequestId());
unwantWorkerTO.setRequiredWorkers(spec.getRequiredWorkers());
unwantWorkerTO.setWorkerAddress(workerAddress);
WorkerDAO workerDAO = BrokerDAOFactory.getInstance().getWorkerDAO();
String workerPublicKey = workerDAO.getWorkerPublicKey(workerAddress);
unwantWorkerTO.setWorkerPublicKey(workerPublicKey);
responses.add(unwantWorkerTO);
ReleaseResponseTO releaseTO = new ReleaseResponseTO();
releaseTO.setStubAddress(workerAddress);
responses.add(releaseTO);
workerDAO.removeWorker(workerAddress);
WorkerInfo.getInstance().removeWorker(workerEntry.getServiceID().getContainerID().toString());
}
private void finishRequests(Job job, List<IResponseTO> responses) {
RequestSpecification spec = null;
for (Request request : job.getRequests()) {
spec = request.getSpecification();
String peerAddress = StringUtil.deploymentIDToAddress(request.getPeerID());
FinishRequestResponseTO to = new FinishRequestResponseTO();
to.setJobID(Integer.parseInt("" + spec.getJobId()));
to.setJobSpec(spec.getJobSpecification());
to.setMaxFails(spec.getMaxFails());
to.setMaxReplicas(spec.getMaxReplicas());
to.setPeerAddress(peerAddress);
to.setRequestID(spec.getRequestId());
to.setRequiredWorkers(spec.getRequiredWorkers());
responses.add(to);
}
job.finishRequests();
}
public void executionFailed(GridProcess execution, List<IResponseTO> responses) {
reportReplicaAccounting(execution, responses);
Job job = execution.getJob();
GridProcessExecutionResult executionResult = execution.getResult();
try {
job.newReplicaResult(
executionResult,
GridProcessState.FAILED,
verifyFailure(execution.getTask(), GridProcessState.FAILED),
canReplicate(execution.getTask()));
} catch (IllegalResultException e) {
responses.add(
new LoggerResponseTO(
"Illegal result on replica " + execution.getState() + " : " + e.getMessage(),
LoggerResponseTO.ERROR));
}
GridProcessHandle handle = executionResult.getReplicaHandle();
WorkerEntry workerEntry = execution.getWorkerEntry();
workerEntry.deallocate();
GridProcessErrorTypes type = null;
if (executionResult != null && executionResult.getExecutionError() != null) {
type = executionResult.getExecutionError().getType();
}
boolean enteredTaskBlacklist = executionFailedOnWorker(workerEntry, type, execution, responses);
if (enteredTaskBlacklist) {
if (!isWorkerNeeded(workerEntry, execution)) {
unwantWorker(job, workerEntry, responses);
}
} else {
disposeWorker(workerEntry, responses);
}
boolean hasJobEnded = hasJobEnded(job);
String executorMsg = "";
if (executionResult != null
&& executionResult.getExecutionError() != null
&& executionResult.getExecutionError().getErrorCause() != null) {
executorMsg = executionResult.getExecutionError().getErrorCause().toString();
}
responses.add(
new LoggerResponseTO(
"Grid process "
+ execution.getState()
+ " "
+ handle
+ ". Job ended: "
+ hasJobEnded
+ " "
+ executorMsg
+ ".",
LoggerResponseTO.DEBUG));
if (hasJobEnded) {
finishJob(execution.getJob(), responses);
}
if (!isJobSatisfied(job) && !hasJobEnded) {
Request request = execution.getJob().getRequest(workerEntry.getRequestID());
if (request != null) {
request.setPaused(false);
}
ResumeRequestResponseTO to = new ResumeRequestResponseTO();
to.setPeerAddress(StringUtil.deploymentIDToAddress(workerEntry.getPeerID()));
to.setRequestID(workerEntry.getRequestID());
responses.add(to);
}
updateScheduler(responses);
}
private void reportReplicaAccounting(GridProcess process, List<IResponseTO> responses) {
GridProcessAccounting accounting = setAccountingFields(process);
accounting.setTransfersProgress(convertTransfer(process.getTransfersProgress()));
String peerID = process.getWorkerProviderID();
String peerAddress = StringUtil.deploymentIDToAddress(peerID);
ReportReplicaAccountingResponseTO to = new ReportReplicaAccountingResponseTO();
to.setCreationTime(accounting.getCreationTime());
to.setErrorCause(accounting.getErrorCause());
to.setExecutionErrorType(accounting.getExecutionErrorType());
to.setExitValue(accounting.getExitValue());
to.setFinalBeginning(accounting.getFinalBeginning());
to.setFinalEnd(accounting.getFinalEnd());
to.setInitBeginning(accounting.getInitBeginning());
to.setInitEnd(accounting.getInitEnd());
to.setJobID(process.getJobId());
to.setLatestPhase(accounting.getLatestPhase());
to.setMaxFails(accounting.getMaxFails());
to.setMaxReplicas(accounting.getMaxReplicas());
to.setPeerAddress(peerAddress);
to.setRemoteBeginning(accounting.getRemoteBeginning());
to.setRemoteEnd(accounting.getRemoteEnd());
to.setRequestID(accounting.getRequestId());
to.setRequiredWorkers(accounting.getRequiredWorkers());
to.setSabotageCheck(accounting.getSabotageCheck());
to.setState(accounting.getState().name());
to.setStderr(accounting.getStderr());
to.setStdout(accounting.getStdout());
to.setTaskSequenceNumber(accounting.getTaskSequenceNumber());
to.setGridProcessSequenceNumber(accounting.getGridProcessSequenceNumber());
to.setWorkerID(accounting.getWorkerID());
to.setWorkerPK(accounting.getWorkerPublicKey());
String workerAddress = StringUtil.deploymentIDToAddress(accounting.getWorkerID());
WorkerSpecification workerSpec =
BrokerDAOFactory.getInstance().getWorkerDAO().getWorkerSpec(workerAddress);
to.setWorkerSpec(workerSpec);
to.setGetOperationsList(
fillFinalGetOperations(
accounting.getFinalCommands(),
process.getTask(),
process.getId(),
process.getWorkerEntry().getWorkerID(),
accounting.getRequestId()));
to.setInitOperationsList(
fillInitGetOperations(
accounting.getInitCommands(),
process.getTask(),
process.getId(),
process.getWorkerEntry().getWorkerID(),
accounting.getRequestId()));
to.setPeerBalancesList(fillPeerBalances(accounting.getAccountings().getBalances()));
to.setTransferProgressList(
fillTransferProgress(accounting.getTransfersProgress(), "" + process.getId()));
responses.add(to);
}