private void updateExecution(
GridProcess execution, GridProcessState state, List<IResponseTO> responses) {
try {
execution
.getJob()
.newReplicaResult(
execution.getResult(),
state,
verifyFailure(execution.getTask(), state),
canReplicate(execution.getTask()));
} catch (IllegalResultException e) {
responses.add(
new LoggerResponseTO(
"Illegal result on replicaEnded: " + e.getMessage(), LoggerResponseTO.ERROR));
}
if (state.equals(GridProcessState.FINISHED)) {
abortReplicaSisters(execution, responses);
}
WorkerEntry workerEntry = execution.getWorkerEntry();
workerEntry.deallocate();
if (!isWorkerNeeded(workerEntry, execution)) {
disposeWorker(workerEntry, responses);
}
}
public boolean executionFailedOnWorker(
WorkerEntry workerEntry,
GridProcessErrorTypes type,
GridProcess execution,
List<IResponseTO> responses) {
if (workerEntry != null && type != null) {
if (type.blackListError()) {
int taskid = execution.getTaskId();
responses.add(
new LoggerResponseTO(
"Adding to blacklist. Task: " + taskid + ", Worker: " + workerEntry.getWorkerID(),
LoggerResponseTO.DEBUG));
workerEntry.addBlacklistedTask(taskid);
// a sabotage error causes a immediately job blacklist entry
if (type.equals(GridProcessErrorTypes.SABOTAGE_ERROR)) {
saboteurs.add(workerEntry);
}
return true;
}
}
return false;
}
public void disposeWorker(WorkerEntry workerEntry, List<IResponseTO> responses) {
ServiceID serviceID = workerEntry.getServiceID();
responses.add(new LoggerResponseTO("Worker dispose: " + serviceID, LoggerResponseTO.DEBUG));
workerEntry.dispose();
WorkerDAO workerDAO = BrokerDAOFactory.getInstance().getWorkerDAO();
String wAddress = StringUtil.deploymentIDToAddress(workerEntry.getWorkerID());
DisposeWorkerResponseTO disposeTO = new DisposeWorkerResponseTO();
disposeTO.setPeerAddress(StringUtil.deploymentIDToAddress(workerEntry.getPeerID()));
disposeTO.setWorkerAddress(wAddress);
disposeTO.setWorkerPublicKey(workerDAO.getWorkerPublicKey(wAddress));
responses.add(disposeTO);
responses.add(new LoggerResponseTO("Stub to be released: " + wAddress, LoggerResponseTO.INFO));
ReleaseResponseTO releaseTO = new ReleaseResponseTO();
releaseTO.setStubAddress(wAddress);
responses.add(releaseTO);
workerDAO.removeWorker(wAddress);
WorkerInfo.getInstance().removeWorker(workerEntry.getServiceID().getContainerID().toString());
}
public boolean isASaboteurWorker(ServiceID workerServiceID) {
if (this.saboteurs != null) {
for (WorkerEntry entry : this.saboteurs) {
if (entry.getServiceID().equals(workerServiceID)) {
return true;
}
}
}
return false;
}
public boolean isWorkerUnwanted(Job job, String workerID) {
DeploymentID workerDeploymentID = new DeploymentID(workerID);
ServiceID workerServiceID = workerDeploymentID.getServiceID();
for (WorkerEntry entry : job.getBlackListedWorkers()) {
if (entry.getServiceID().equals(workerServiceID)) {
return true;
}
}
return isASaboteurWorker(workerServiceID);
}
protected int getRemainingBlacklistFails(WorkerEntry workerEntry, Job job) {
if (isASaboteurWorker(workerEntry.getServiceID())) {
return 0;
}
if (workerEntry != null) {
/*int jobSize = Math.min(job.getSpec().getTaskSpecs().size(), this.maxBlFails);
return Math.max( 0, jobSize - workerEntry.getNumberOfBlacklistedTasks() );*/
return Math.max(0, this.maxBlFails - workerEntry.getNumberOfBlacklistedTasks());
}
return -1;
}
private boolean createAndAllocateExecution(Job job, Task task, WorkerEntry chosenWorker) {
GridProcess replica = null;
if (canReplicate(task)) {
replica = job.createAndAllocateExecution(task.getTaskid(), chosenWorker);
replica.setRunningState(stateMachine.getInitialState());
}
if (replica != null) {
chosenWorker.allocate(replica);
WorkerEntry worker =
WorkerInfo.getInstance()
.getWorker(chosenWorker.getServiceID().getContainerID().toString());
worker.allocate(replica);
return true;
}
return false;
}
/**
* Mark's this <code>GridProcess</code> as running, in other words this replica will change state
* to <code>GridProcessState.RUNNING</code> if it is in <code>GridProcessState.UNSTARTED</code>.
*
* @param chosenWorker Spec of the worker that will run this replica.
*/
public void allocate(WorkerEntry chosenWorker) {
if (!GridProcessState.UNSTARTED.equals(this.state)) {
throw new IllegalResultException(
"This replica is already running or has already finished execution", replicaHandle);
}
this.workerEntry = chosenWorker;
RequestSpecification requestSpec = workerEntry.getRequestSpecification();
if (requestSpec != null) {
this.replicaAccounting =
new GridProcessAccounting(
requestSpec.getRequestId(),
requestSpec.getJobId(),
requestSpec.getRequiredWorkers(),
requestSpec.getMaxFails(),
requestSpec.getMaxReplicas(),
getWorkerEntry().getWorkerID(),
getWorkerEntry().getWorkerPublicKey(),
workerEntry.getWorkerSpecification());
}
}
/**
* @param job
* @param workerEntry
*/
private void unwantWorker(Job job, WorkerEntry workerEntry, List<IResponseTO> responses) {
responses.add(
new LoggerResponseTO(
"Worker unwanted: " + workerEntry.getWorkerID(), LoggerResponseTO.DEBUG));
job.unwantWorker(workerEntry);
workerEntry.unwant();
RequestSpecification spec = workerEntry.getRequestSpecification();
String peerAddress = StringUtil.deploymentIDToAddress(workerEntry.getPeerID());
String workerAddress = StringUtil.deploymentIDToAddress(workerEntry.getWorkerID());
Integer jobId = Integer.parseInt("" + spec.getJobId());
JobSpecification jobSpec = BrokerDAOFactory.getInstance().getJobDAO().getJobSpec(jobId);
UnwantWorkerResponseTO unwantWorkerTO = new UnwantWorkerResponseTO();
unwantWorkerTO.setJobID(jobId);
unwantWorkerTO.setJobSpec(jobSpec);
unwantWorkerTO.setMaxFails(spec.getMaxFails());
unwantWorkerTO.setMaxReplicas(spec.getMaxReplicas());
unwantWorkerTO.setPeerAddress(peerAddress);
unwantWorkerTO.setRequestID(spec.getRequestId());
unwantWorkerTO.setRequiredWorkers(spec.getRequiredWorkers());
unwantWorkerTO.setWorkerAddress(workerAddress);
WorkerDAO workerDAO = BrokerDAOFactory.getInstance().getWorkerDAO();
String workerPublicKey = workerDAO.getWorkerPublicKey(workerAddress);
unwantWorkerTO.setWorkerPublicKey(workerPublicKey);
responses.add(unwantWorkerTO);
ReleaseResponseTO releaseTO = new ReleaseResponseTO();
releaseTO.setStubAddress(workerAddress);
responses.add(releaseTO);
workerDAO.removeWorker(workerAddress);
WorkerInfo.getInstance().removeWorker(workerEntry.getServiceID().getContainerID().toString());
}
public void executionFailed(GridProcess execution, List<IResponseTO> responses) {
reportReplicaAccounting(execution, responses);
Job job = execution.getJob();
GridProcessExecutionResult executionResult = execution.getResult();
try {
job.newReplicaResult(
executionResult,
GridProcessState.FAILED,
verifyFailure(execution.getTask(), GridProcessState.FAILED),
canReplicate(execution.getTask()));
} catch (IllegalResultException e) {
responses.add(
new LoggerResponseTO(
"Illegal result on replica " + execution.getState() + " : " + e.getMessage(),
LoggerResponseTO.ERROR));
}
GridProcessHandle handle = executionResult.getReplicaHandle();
WorkerEntry workerEntry = execution.getWorkerEntry();
workerEntry.deallocate();
GridProcessErrorTypes type = null;
if (executionResult != null && executionResult.getExecutionError() != null) {
type = executionResult.getExecutionError().getType();
}
boolean enteredTaskBlacklist = executionFailedOnWorker(workerEntry, type, execution, responses);
if (enteredTaskBlacklist) {
if (!isWorkerNeeded(workerEntry, execution)) {
unwantWorker(job, workerEntry, responses);
}
} else {
disposeWorker(workerEntry, responses);
}
boolean hasJobEnded = hasJobEnded(job);
String executorMsg = "";
if (executionResult != null
&& executionResult.getExecutionError() != null
&& executionResult.getExecutionError().getErrorCause() != null) {
executorMsg = executionResult.getExecutionError().getErrorCause().toString();
}
responses.add(
new LoggerResponseTO(
"Grid process "
+ execution.getState()
+ " "
+ handle
+ ". Job ended: "
+ hasJobEnded
+ " "
+ executorMsg
+ ".",
LoggerResponseTO.DEBUG));
if (hasJobEnded) {
finishJob(execution.getJob(), responses);
}
if (!isJobSatisfied(job) && !hasJobEnded) {
Request request = execution.getJob().getRequest(workerEntry.getRequestID());
if (request != null) {
request.setPaused(false);
}
ResumeRequestResponseTO to = new ResumeRequestResponseTO();
to.setPeerAddress(StringUtil.deploymentIDToAddress(workerEntry.getPeerID()));
to.setRequestID(workerEntry.getRequestID());
responses.add(to);
}
updateScheduler(responses);
}
public JobWorkerStatus getCompleteStatus() {
Map<Integer, Job> jobsMap = JobInfo.getInstance().getJobs();
Map<Integer, Set<WorkerEntry>> workersByJob = CommonUtils.createMap();
JobStatusInfo jobInfo = null;
List<TaskStatusInfo> tasksList = null;
Map<Integer, JobStatusInfo> jobs = CommonUtils.createSerializableMap();
// Jobs
for (Job job : jobsMap.values()) {
Set<WorkerEntry> workers = new LinkedHashSet<WorkerEntry>();
tasksList = new ArrayList<TaskStatusInfo>();
for (Task task : job.getTasks()) {
tasksList.add(fillTask(task));
for (GridProcess gridProcess : task.getGridProcesses()) {
if (gridProcess.getState() == GridProcessState.RUNNING)
workers.add(gridProcess.getWorkerEntry());
}
}
jobInfo =
new JobStatusInfo(
job.getJobId(),
job.getSpec(),
UtilConverter.getJobState(job.getState()),
tasksList,
job.getCreationTime(),
job.getFinalizationTime());
jobs.put(jobInfo.getJobId(), jobInfo);
if (job.isRunning()) {
workersByJob.put(job.getJobId(), workers);
}
}
Map<Integer, WorkerStatusInfo[]> workers = CommonUtils.createSerializableMap();
WorkerStatusInfo[] workerList = null;
for (Entry<Integer, Set<WorkerEntry>> entry : workersByJob.entrySet()) {
workerList = workers.get(entry.getKey());
if (workerList == null) {
workerList = new WorkerStatusInfo[entry.getValue().size()];
workers.put(entry.getKey(), workerList);
}
int i = 0;
for (WorkerEntry workerEntry : entry.getValue()) {
GridProcessHandle handle = null;
String state = null;
if (workerEntry.getGridProcess() != null) {
handle = workerEntry.getGridProcess().getHandle();
state = workerEntry.getGridProcess().getState().toString();
}
workerList[i] =
new WorkerStatusInfo(
workerEntry.getWorkerSpecification(), handle, workerEntry.getWorkerID(), state);
i++;
}
}
JobWorkerStatus status = new JobWorkerStatus(jobs, workers);
return status;
}
public int compare(WorkerEntry o1, WorkerEntry o2) {
return workerSpecAnnotationsComparator.compare(
o1.getWorkerSpecification(), o2.getWorkerSpecification());
}