コード例 #1
0
  /**
   * Create a new internal job with the given job (user).
   *
   * @param job the user job that will be used to create the internal job.
   * @return the created internal job.
   * @throws JobCreationException an exception if the factory cannot create the given job.
   */
  public static InternalJob createJob(Job job, Credentials cred) throws JobCreationException {
    InternalJob iJob;

    if (logger.isDebugEnabled()) {
      logger.debug("Create job '" + job.getName() + "' - " + job.getClass().getName());
    }

    switch (job.getType()) {
      case PARAMETER_SWEEPING:
        logger.error("The type of the given job is not yet implemented !");
        throw new JobCreationException("The type of the given job is not yet implemented !");
      case TASKSFLOW:
        iJob = createJob((TaskFlowJob) job);
        break;
      default:
        logger.error("The type of the given job is unknown !");
        throw new JobCreationException("The type of the given job is unknown !");
    }

    try {
      // set the job common properties
      iJob.setCredentials(cred);
      setJobCommonProperties(job, iJob);
      return iJob;
    } catch (Exception e) {
      logger.error("", e);
      throw new InternalException("Error while creating the internalJob !", e);
    }
  }
コード例 #2
0
  /**
   * This method will create a remote folder for output of this job and update the outputSpace job
   * property. If the localOutputFolder parameter is null, or pull_url no action will be performed
   * concerning this job's output.
   *
   * <p>We suppose there is file storage accessible by the client application as well as the tasks
   * on the computation nodes.
   *
   * <p>This storage could be different for input and for output.
   *
   * <p>This output storage can be accessed, by the client application, using the pull_url and by
   * the tasks on the nodes using the job's output space url.
   *
   * <p>Prepare Output Data Transfer
   *
   * <p>A folder will be created at pull_url/NewFolder/output (which, from the nodes side, is
   * job.OutputSpace/NewFolder/output).
   *
   * <p>The OutputSpace property of the job will be changed to the new location. job.OutputSpace =
   * job.OutputSpace/NewFolder/output
   *
   * <p>A generic information will be attached to the job containing the local output folder path
   *
   * @param job
   * @param localOutputFolder path to the output folder on local machine if null, no actions will be
   *     performed concerning the output data for this job
   * @param pull_url - the url where the data is to be retrieved after the job is finished
   * @param newFolderName name of the folder to be used for pushing the output
   * @return a String representing the updated value of the pull_url
   * @throws FileSystemException
   */
  private String prepareJobOutput(
      Job job, String localOutputFolder, String pull_url, String newFolderName)
      throws FileSystemException {
    // if the job defines an output space
    // and the localOutputFolder is not null
    // create a remote folder for the output data
    // and update the OutputSpace property of the job to reference that
    // folder
    String outputSpace_url = job.getOutputSpace();
    String pull_url_updated = "";

    // the output folder, on the remote output space, relative to the root
    // url
    String outputFolder = "";

    if ((localOutputFolder != null)
        && (outputSpace_url != null)
        && (!outputSpace_url.equals(""))
        && (pull_url != null)) {
      outputFolder = newFolderName + "/output";

      pull_url_updated = pull_url + "//" + outputFolder;
      String outputSpace_url_updated = outputSpace_url + "//" + outputFolder;
      createFolder(pull_url_updated);

      job.setOutputSpace(outputSpace_url_updated);
      job.addGenericInformation(
          GENERIC_INFO_OUTPUT_FOLDER_PROPERTY_NAME, new File(localOutputFolder).getAbsolutePath());
      job.addGenericInformation(GENERIC_INFO_PULL_URL_PROPERTY_NAME, pull_url_updated);
    }

    return pull_url_updated;
  }
コード例 #3
0
 /**
  * Set some properties between the user Job and internal Job.
  *
  * @param job the user job.
  * @param jobToSet the internal job to set.
  * @throws IllegalAccessException
  * @throws IllegalArgumentException
  */
 private static void setJobCommonProperties(Job job, InternalJob jobToSet)
     throws IllegalArgumentException, IllegalAccessException {
   autoCopyfields(CommonAttribute.class, job, jobToSet);
   autoCopyfields(Job.class, job, jobToSet);
   // special behavior
   jobToSet.setPriority(job.getPriority());
 }
コード例 #4
0
  /**
   * Set some properties between the user task and internal task.
   *
   * @param task the user task.
   * @param taskToSet the internal task to set.
   * @throws IllegalAccessException
   * @throws IllegalArgumentException
   */
  private static void setTaskCommonProperties(Job userJob, Task task, InternalTask taskToSet)
      throws IllegalArgumentException, IllegalAccessException {
    autoCopyfields(CommonAttribute.class, task, taskToSet);
    autoCopyfields(Task.class, task, taskToSet);

    // special behavior
    if (onErrorPolicyInterpreter.notSetOrNone(task)) {
      taskToSet.setOnTaskError(userJob.getOnTaskErrorProperty().getValue());
    } else {
      taskToSet.setOnTaskError(task.getOnTaskErrorProperty().getValue());
    }
    if (task.getRestartTaskOnErrorProperty().isSet()) {
      taskToSet.setRestartTaskOnError(task.getRestartTaskOnError());
    } else {
      taskToSet.setRestartTaskOnError(userJob.getRestartTaskOnError());
    }
    if (task.getMaxNumberOfExecutionProperty().isSet()) {
      taskToSet.setMaxNumberOfExecution(task.getMaxNumberOfExecution());
    } else {
      taskToSet.setMaxNumberOfExecution(userJob.getMaxNumberOfExecution());
    }
  }
コード例 #5
0
  /**
   * Does the following steps:
   *
   * <ul>
   *   <li>Prepares the temporary folders. See {@link #prepareDataFolderAndUpdateJob(Job, String,
   *       String, String, String)}
   *   <li>pushes all files from localInputFolderPath to the push_url location (see {@link
   *       #pushData(Job, String)})
   *   <li>submits the job to the scheduler
   *   <li>adds the job to the awaited jobs list in order to download the output data when the job
   *       is finished
   * </ul>
   *
   * Note: this method is synchronous. The caller will be blocked until the the data is pushed and
   * the job submitted.
   *
   * @param job job object to be submitted to the Scheduler Server for execution
   * @param localInputFolderPath path to the folder containing the input data for this job
   * @param push_url the url where input data is to be pushed before the job submission
   * @param localOutputFolderPath path to the folder where the output data produced by tasks in this
   *     job should be copied
   * @param pull_url the url where the data is to be retrieved after the job is finished
   * @param localOutputFolderPath
   * @return
   * @throws JobCreationException
   * @throws SubmissionClosedException
   * @throws PermissionException
   * @throws NotConnectedException
   * @throws FileSystemException
   */
  public JobId submit(
      Job job,
      String localInputFolderPath,
      String push_url,
      String localOutputFolderPath,
      String pull_url)
      throws NotConnectedException, PermissionException, SubmissionClosedException,
          JobCreationException, FileSystemException {

    if (((push_url == null) || (push_url.equals("")))
        && ((pull_url == null) || (pull_url.equals("")))) {
      logger.warn(
          "For the job "
              + job.getId()
              + " no push or pull urls are defined. No data will be transfered for this job from the local machine ");
      return super.submit(job);
    }

    String newFolderName = createNewFolderName();
    String push_Url_update = prepareJobInput(job, localInputFolderPath, push_url, newFolderName);
    String pull_url_update = prepareJobOutput(job, localOutputFolderPath, pull_url, newFolderName);

    pushData(job, localInputFolderPath);
    JobId id = super.submit(job);

    AwaitedJob aj =
        new AwaitedJob(
            id.toString(),
            localInputFolderPath,
            job.getInputSpace(),
            push_Url_update,
            localOutputFolderPath,
            job.getOutputSpace(),
            pull_url_update);

    addAwaitedJob(aj);
    return id;
  }
コード例 #6
0
  /**
   * @param job
   * @param inputFolder
   * @return
   * @throws FileSystemException
   */
  protected boolean pushData(Job job, String localInputFolderPath) throws FileSystemException {

    String push_URL = job.getGenericInformations().get(GENERIC_INFO_PUSH_URL_PROPERTY_NAME);

    if ((push_URL == null) || (push_URL.trim().equals(""))) {
      return false;
    } // push inputData

    // TODO - if the copy fails, try to remove the files from the remote
    // folder before throwing an exception
    FileObject remoteFolder = fsManager.resolveFile(push_URL);
    FileObject localfolder = fsManager.resolveFile(localInputFolderPath);
    logger.debug("Pushing files from " + localfolder + " to " + remoteFolder);

    // create the selector
    DSFileSelector fileSelector = new DSFileSelector();

    TaskFlowJob tfj = (TaskFlowJob) job;
    for (Task t : tfj.getTasks()) {
      List<InputSelector> inputFileSelectors = t.getInputFilesList();
      for (InputSelector is : inputFileSelectors) {
        org.ow2.proactive.scheduler.common.task.dataspaces.FileSelector fs = is.getInputFiles();
        if (fs.getIncludes() != null) fileSelector.addIncludes(Arrays.asList(fs.getIncludes()));

        if (fs.getExcludes() != null) fileSelector.addExcludes(Arrays.asList(fs.getExcludes()));
      }
    }

    // We need to check if a pattern exist in both includes and excludes.
    // This may happen if a task one defines, for instance "*.txt" as includes
    // and a task two defines "*.txt" as excludes. In this case we should remove it from the
    // fileSelector's excludes.

    Set<String> includes = fileSelector.getIncludes();
    Set<String> excludes = fileSelector.getExcludes();

    Set<String> intersection = new HashSet<String>(includes);
    intersection.retainAll(excludes);
    excludes.removeAll(intersection);
    fileSelector.setExcludes(excludes);

    remoteFolder.copyFrom(localfolder, fileSelector);

    logger.debug("Finished push operation from " + localfolder + " to " + remoteFolder);
    return true;
  }
コード例 #7
0
  /**
   * Tests start here.
   *
   * @throws Throwable any exception that can be thrown during the test.
   */
  @org.junit.Test
  public void run() throws Throwable {

    String task1Name = "Task1";
    String task2Name = "Task2";

    String taskForked1Name = "Fork1";
    String taskForked2Name = "Fork2";
    TaskFlowJob job =
        (TaskFlowJob)
            JobFactory_stax.getFactory()
                .createJob(new File(jobDescriptor.toURI()).getAbsolutePath());
    if (OperatingSystem.getOperatingSystem() == OperatingSystem.windows) {
      ((NativeTask) job.getTask(task1Name))
          .setCommandLine("cmd", "/C", "ping", "127.0.0.1", "-n", "20", ">", "NUL");
      ((NativeTask) job.getTask(taskForked1Name))
          .setCommandLine("cmd", "/C", "ping", "127.0.0.1", "-n", "20", ">", "NUL");
    }
    JobId id = SchedulerTHelper.submitJob(job);

    SchedulerTHelper.log("Job submitted, id " + id.toString());

    SchedulerTHelper.log("Waiting for jobSubmitted Event");
    Job receivedJob = SchedulerTHelper.waitForEventJobSubmitted(id);
    Assert.assertEquals(receivedJob.getId(), id);

    SchedulerTHelper.log("Waiting for job running");
    JobInfo jInfo = SchedulerTHelper.waitForEventJobRunning(id);
    Assert.assertEquals(jInfo.getJobId(), id);
    Assert.assertEquals(JobStatus.RUNNING, jInfo.getStatus());

    SchedulerTHelper.log("check events for task " + task1Name);
    TaskInfo tInfo = SchedulerTHelper.waitForEventTaskRunning(id, task1Name);
    Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus());
    tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task1Name);
    Assert.assertEquals(TaskStatus.FINISHED, tInfo.getStatus());

    SchedulerTHelper.log("check events for task " + task2Name);
    tInfo = SchedulerTHelper.waitForEventTaskRunning(id, task2Name);
    Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus());
    tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task2Name);
    Assert.assertEquals(TaskStatus.FINISHED, tInfo.getStatus());

    // this task reaches wall time, so finishes with faulty state
    SchedulerTHelper.log("check events for task " + taskForked1Name);
    tInfo = SchedulerTHelper.waitForEventTaskRunning(id, taskForked1Name);
    Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus());
    tInfo = SchedulerTHelper.waitForEventTaskFinished(id, taskForked1Name);
    Assert.assertEquals(TaskStatus.FAULTY, tInfo.getStatus());

    // this task reaches wall time, so finishes with faulty state
    SchedulerTHelper.log("check events for task " + taskForked2Name);
    tInfo = SchedulerTHelper.waitForEventTaskRunning(id, taskForked2Name);
    Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus());
    tInfo = SchedulerTHelper.waitForEventTaskFinished(id, taskForked2Name);
    Assert.assertEquals(TaskStatus.FAULTY, tInfo.getStatus());

    SchedulerTHelper.log("Waiting for job finished");
    jInfo = SchedulerTHelper.waitForEventJobFinished(id);
    Assert.assertEquals(jInfo.getJobId(), id);
    Assert.assertEquals(JobStatus.FINISHED, jInfo.getStatus());

    // check result are not null
    JobResult res = SchedulerTHelper.getJobResult(id);
    Assert.assertTrue(res.hadException());

    Assert.assertFalse(res.getResult(task1Name).hadException());
    Assert.assertNull(res.getResult(task1Name).getException());

    Assert.assertFalse(res.getResult(task2Name).hadException());
    Assert.assertNull(res.getResult(task2Name).getException());

    Assert.assertFalse(res.getResult(taskForked1Name).hadException());
    Assert.assertNull(res.getResult(taskForked1Name).getException());

    Assert.assertTrue(res.getResult(taskForked2Name).hadException());
    Assert.assertNotNull(res.getResult(taskForked2Name).getException());

    // remove job
    SchedulerTHelper.removeJob(id);
    SchedulerTHelper.waitForEventJobRemoved(id);
  }