private void updateExecution(
      GridProcess execution, GridProcessState state, List<IResponseTO> responses) {
    try {
      execution
          .getJob()
          .newReplicaResult(
              execution.getResult(),
              state,
              verifyFailure(execution.getTask(), state),
              canReplicate(execution.getTask()));
    } catch (IllegalResultException e) {

      responses.add(
          new LoggerResponseTO(
              "Illegal result on replicaEnded: " + e.getMessage(), LoggerResponseTO.ERROR));
    }

    if (state.equals(GridProcessState.FINISHED)) {
      abortReplicaSisters(execution, responses);
    }

    WorkerEntry workerEntry = execution.getWorkerEntry();
    workerEntry.deallocate();

    if (!isWorkerNeeded(workerEntry, execution)) {
      disposeWorker(workerEntry, responses);
    }
  }
  public boolean executionFailedOnWorker(
      WorkerEntry workerEntry,
      GridProcessErrorTypes type,
      GridProcess execution,
      List<IResponseTO> responses) {

    if (workerEntry != null && type != null) {
      if (type.blackListError()) {

        int taskid = execution.getTaskId();

        responses.add(
            new LoggerResponseTO(
                "Adding to blacklist. Task: " + taskid + ", Worker: " + workerEntry.getWorkerID(),
                LoggerResponseTO.DEBUG));

        workerEntry.addBlacklistedTask(taskid);

        // a sabotage error causes a immediately job blacklist entry
        if (type.equals(GridProcessErrorTypes.SABOTAGE_ERROR)) {
          saboteurs.add(workerEntry);
        }
        return true;
      }
    }
    return false;
  }
  public void disposeWorker(WorkerEntry workerEntry, List<IResponseTO> responses) {
    ServiceID serviceID = workerEntry.getServiceID();

    responses.add(new LoggerResponseTO("Worker dispose: " + serviceID, LoggerResponseTO.DEBUG));

    workerEntry.dispose();

    WorkerDAO workerDAO = BrokerDAOFactory.getInstance().getWorkerDAO();
    String wAddress = StringUtil.deploymentIDToAddress(workerEntry.getWorkerID());

    DisposeWorkerResponseTO disposeTO = new DisposeWorkerResponseTO();
    disposeTO.setPeerAddress(StringUtil.deploymentIDToAddress(workerEntry.getPeerID()));
    disposeTO.setWorkerAddress(wAddress);
    disposeTO.setWorkerPublicKey(workerDAO.getWorkerPublicKey(wAddress));
    responses.add(disposeTO);

    responses.add(new LoggerResponseTO("Stub to be released: " + wAddress, LoggerResponseTO.INFO));

    ReleaseResponseTO releaseTO = new ReleaseResponseTO();
    releaseTO.setStubAddress(wAddress);
    responses.add(releaseTO);

    workerDAO.removeWorker(wAddress);
    WorkerInfo.getInstance().removeWorker(workerEntry.getServiceID().getContainerID().toString());
  }
  public boolean isASaboteurWorker(ServiceID workerServiceID) {

    if (this.saboteurs != null) {
      for (WorkerEntry entry : this.saboteurs) {
        if (entry.getServiceID().equals(workerServiceID)) {
          return true;
        }
      }
    }
    return false;
  }
  public boolean isWorkerUnwanted(Job job, String workerID) {

    DeploymentID workerDeploymentID = new DeploymentID(workerID);
    ServiceID workerServiceID = workerDeploymentID.getServiceID();

    for (WorkerEntry entry : job.getBlackListedWorkers()) {
      if (entry.getServiceID().equals(workerServiceID)) {
        return true;
      }
    }

    return isASaboteurWorker(workerServiceID);
  }
  protected int getRemainingBlacklistFails(WorkerEntry workerEntry, Job job) {

    if (isASaboteurWorker(workerEntry.getServiceID())) {
      return 0;
    }

    if (workerEntry != null) {
      /*int jobSize = Math.min(job.getSpec().getTaskSpecs().size(), this.maxBlFails);
      return Math.max( 0, jobSize - workerEntry.getNumberOfBlacklistedTasks() );*/

      return Math.max(0, this.maxBlFails - workerEntry.getNumberOfBlacklistedTasks());
    }

    return -1;
  }
  private boolean createAndAllocateExecution(Job job, Task task, WorkerEntry chosenWorker) {

    GridProcess replica = null;
    if (canReplicate(task)) {
      replica = job.createAndAllocateExecution(task.getTaskid(), chosenWorker);
      replica.setRunningState(stateMachine.getInitialState());
    }

    if (replica != null) {
      chosenWorker.allocate(replica);

      WorkerEntry worker =
          WorkerInfo.getInstance()
              .getWorker(chosenWorker.getServiceID().getContainerID().toString());
      worker.allocate(replica);
      return true;
    }

    return false;
  }
Ejemplo n.º 8
0
  /**
   * Mark's this <code>GridProcess</code> as running, in other words this replica will change state
   * to <code>GridProcessState.RUNNING</code> if it is in <code>GridProcessState.UNSTARTED</code>.
   *
   * @param chosenWorker Spec of the worker that will run this replica.
   */
  public void allocate(WorkerEntry chosenWorker) {

    if (!GridProcessState.UNSTARTED.equals(this.state)) {
      throw new IllegalResultException(
          "This replica is already running or has already finished execution", replicaHandle);
    }

    this.workerEntry = chosenWorker;
    RequestSpecification requestSpec = workerEntry.getRequestSpecification();

    if (requestSpec != null) {
      this.replicaAccounting =
          new GridProcessAccounting(
              requestSpec.getRequestId(),
              requestSpec.getJobId(),
              requestSpec.getRequiredWorkers(),
              requestSpec.getMaxFails(),
              requestSpec.getMaxReplicas(),
              getWorkerEntry().getWorkerID(),
              getWorkerEntry().getWorkerPublicKey(),
              workerEntry.getWorkerSpecification());
    }
  }
  /**
   * @param job
   * @param workerEntry
   */
  private void unwantWorker(Job job, WorkerEntry workerEntry, List<IResponseTO> responses) {

    responses.add(
        new LoggerResponseTO(
            "Worker unwanted: " + workerEntry.getWorkerID(), LoggerResponseTO.DEBUG));

    job.unwantWorker(workerEntry);
    workerEntry.unwant();

    RequestSpecification spec = workerEntry.getRequestSpecification();

    String peerAddress = StringUtil.deploymentIDToAddress(workerEntry.getPeerID());
    String workerAddress = StringUtil.deploymentIDToAddress(workerEntry.getWorkerID());
    Integer jobId = Integer.parseInt("" + spec.getJobId());

    JobSpecification jobSpec = BrokerDAOFactory.getInstance().getJobDAO().getJobSpec(jobId);

    UnwantWorkerResponseTO unwantWorkerTO = new UnwantWorkerResponseTO();
    unwantWorkerTO.setJobID(jobId);
    unwantWorkerTO.setJobSpec(jobSpec);
    unwantWorkerTO.setMaxFails(spec.getMaxFails());
    unwantWorkerTO.setMaxReplicas(spec.getMaxReplicas());
    unwantWorkerTO.setPeerAddress(peerAddress);
    unwantWorkerTO.setRequestID(spec.getRequestId());
    unwantWorkerTO.setRequiredWorkers(spec.getRequiredWorkers());

    unwantWorkerTO.setWorkerAddress(workerAddress);

    WorkerDAO workerDAO = BrokerDAOFactory.getInstance().getWorkerDAO();

    String workerPublicKey = workerDAO.getWorkerPublicKey(workerAddress);
    unwantWorkerTO.setWorkerPublicKey(workerPublicKey);

    responses.add(unwantWorkerTO);

    ReleaseResponseTO releaseTO = new ReleaseResponseTO();
    releaseTO.setStubAddress(workerAddress);

    responses.add(releaseTO);

    workerDAO.removeWorker(workerAddress);

    WorkerInfo.getInstance().removeWorker(workerEntry.getServiceID().getContainerID().toString());
  }
  public void executionFailed(GridProcess execution, List<IResponseTO> responses) {

    reportReplicaAccounting(execution, responses);

    Job job = execution.getJob();
    GridProcessExecutionResult executionResult = execution.getResult();

    try {
      job.newReplicaResult(
          executionResult,
          GridProcessState.FAILED,
          verifyFailure(execution.getTask(), GridProcessState.FAILED),
          canReplicate(execution.getTask()));
    } catch (IllegalResultException e) {

      responses.add(
          new LoggerResponseTO(
              "Illegal result on replica " + execution.getState() + " : " + e.getMessage(),
              LoggerResponseTO.ERROR));
    }

    GridProcessHandle handle = executionResult.getReplicaHandle();

    WorkerEntry workerEntry = execution.getWorkerEntry();
    workerEntry.deallocate();

    GridProcessErrorTypes type = null;

    if (executionResult != null && executionResult.getExecutionError() != null) {
      type = executionResult.getExecutionError().getType();
    }

    boolean enteredTaskBlacklist = executionFailedOnWorker(workerEntry, type, execution, responses);

    if (enteredTaskBlacklist) {
      if (!isWorkerNeeded(workerEntry, execution)) {
        unwantWorker(job, workerEntry, responses);
      }
    } else {
      disposeWorker(workerEntry, responses);
    }

    boolean hasJobEnded = hasJobEnded(job);

    String executorMsg = "";

    if (executionResult != null
        && executionResult.getExecutionError() != null
        && executionResult.getExecutionError().getErrorCause() != null) {
      executorMsg = executionResult.getExecutionError().getErrorCause().toString();
    }

    responses.add(
        new LoggerResponseTO(
            "Grid process "
                + execution.getState()
                + " "
                + handle
                + ". Job ended: "
                + hasJobEnded
                + " "
                + executorMsg
                + ".",
            LoggerResponseTO.DEBUG));

    if (hasJobEnded) {
      finishJob(execution.getJob(), responses);
    }

    if (!isJobSatisfied(job) && !hasJobEnded) {

      Request request = execution.getJob().getRequest(workerEntry.getRequestID());
      if (request != null) {
        request.setPaused(false);
      }

      ResumeRequestResponseTO to = new ResumeRequestResponseTO();

      to.setPeerAddress(StringUtil.deploymentIDToAddress(workerEntry.getPeerID()));
      to.setRequestID(workerEntry.getRequestID());

      responses.add(to);
    }

    updateScheduler(responses);
  }
  public JobWorkerStatus getCompleteStatus() {

    Map<Integer, Job> jobsMap = JobInfo.getInstance().getJobs();
    Map<Integer, Set<WorkerEntry>> workersByJob = CommonUtils.createMap();

    JobStatusInfo jobInfo = null;
    List<TaskStatusInfo> tasksList = null;
    Map<Integer, JobStatusInfo> jobs = CommonUtils.createSerializableMap();

    // Jobs
    for (Job job : jobsMap.values()) {
      Set<WorkerEntry> workers = new LinkedHashSet<WorkerEntry>();
      tasksList = new ArrayList<TaskStatusInfo>();

      for (Task task : job.getTasks()) {
        tasksList.add(fillTask(task));
        for (GridProcess gridProcess : task.getGridProcesses()) {
          if (gridProcess.getState() == GridProcessState.RUNNING)
            workers.add(gridProcess.getWorkerEntry());
        }
      }

      jobInfo =
          new JobStatusInfo(
              job.getJobId(),
              job.getSpec(),
              UtilConverter.getJobState(job.getState()),
              tasksList,
              job.getCreationTime(),
              job.getFinalizationTime());

      jobs.put(jobInfo.getJobId(), jobInfo);

      if (job.isRunning()) {
        workersByJob.put(job.getJobId(), workers);
      }
    }

    Map<Integer, WorkerStatusInfo[]> workers = CommonUtils.createSerializableMap();

    WorkerStatusInfo[] workerList = null;
    for (Entry<Integer, Set<WorkerEntry>> entry : workersByJob.entrySet()) {

      workerList = workers.get(entry.getKey());
      if (workerList == null) {
        workerList = new WorkerStatusInfo[entry.getValue().size()];
        workers.put(entry.getKey(), workerList);
      }

      int i = 0;
      for (WorkerEntry workerEntry : entry.getValue()) {

        GridProcessHandle handle = null;
        String state = null;

        if (workerEntry.getGridProcess() != null) {
          handle = workerEntry.getGridProcess().getHandle();
          state = workerEntry.getGridProcess().getState().toString();
        }

        workerList[i] =
            new WorkerStatusInfo(
                workerEntry.getWorkerSpecification(), handle, workerEntry.getWorkerID(), state);
        i++;
      }
    }

    JobWorkerStatus status = new JobWorkerStatus(jobs, workers);

    return status;
  }
 public int compare(WorkerEntry o1, WorkerEntry o2) {
   return workerSpecAnnotationsComparator.compare(
       o1.getWorkerSpecification(), o2.getWorkerSpecification());
 }