/** * Create a new internal job with the given job (user). * * @param job the user job that will be used to create the internal job. * @return the created internal job. * @throws JobCreationException an exception if the factory cannot create the given job. */ public static InternalJob createJob(Job job, Credentials cred) throws JobCreationException { InternalJob iJob; if (logger.isDebugEnabled()) { logger.debug("Create job '" + job.getName() + "' - " + job.getClass().getName()); } switch (job.getType()) { case PARAMETER_SWEEPING: logger.error("The type of the given job is not yet implemented !"); throw new JobCreationException("The type of the given job is not yet implemented !"); case TASKSFLOW: iJob = createJob((TaskFlowJob) job); break; default: logger.error("The type of the given job is unknown !"); throw new JobCreationException("The type of the given job is unknown !"); } try { // set the job common properties iJob.setCredentials(cred); setJobCommonProperties(job, iJob); return iJob; } catch (Exception e) { logger.error("", e); throw new InternalException("Error while creating the internalJob !", e); } }
/** * This method will create a remote folder for output of this job and update the outputSpace job * property. If the localOutputFolder parameter is null, or pull_url no action will be performed * concerning this job's output. * * <p>We suppose there is file storage accessible by the client application as well as the tasks * on the computation nodes. * * <p>This storage could be different for input and for output. * * <p>This output storage can be accessed, by the client application, using the pull_url and by * the tasks on the nodes using the job's output space url. * * <p>Prepare Output Data Transfer * * <p>A folder will be created at pull_url/NewFolder/output (which, from the nodes side, is * job.OutputSpace/NewFolder/output). * * <p>The OutputSpace property of the job will be changed to the new location. job.OutputSpace = * job.OutputSpace/NewFolder/output * * <p>A generic information will be attached to the job containing the local output folder path * * @param job * @param localOutputFolder path to the output folder on local machine if null, no actions will be * performed concerning the output data for this job * @param pull_url - the url where the data is to be retrieved after the job is finished * @param newFolderName name of the folder to be used for pushing the output * @return a String representing the updated value of the pull_url * @throws FileSystemException */ private String prepareJobOutput( Job job, String localOutputFolder, String pull_url, String newFolderName) throws FileSystemException { // if the job defines an output space // and the localOutputFolder is not null // create a remote folder for the output data // and update the OutputSpace property of the job to reference that // folder String outputSpace_url = job.getOutputSpace(); String pull_url_updated = ""; // the output folder, on the remote output space, relative to the root // url String outputFolder = ""; if ((localOutputFolder != null) && (outputSpace_url != null) && (!outputSpace_url.equals("")) && (pull_url != null)) { outputFolder = newFolderName + "/output"; pull_url_updated = pull_url + "//" + outputFolder; String outputSpace_url_updated = outputSpace_url + "//" + outputFolder; createFolder(pull_url_updated); job.setOutputSpace(outputSpace_url_updated); job.addGenericInformation( GENERIC_INFO_OUTPUT_FOLDER_PROPERTY_NAME, new File(localOutputFolder).getAbsolutePath()); job.addGenericInformation(GENERIC_INFO_PULL_URL_PROPERTY_NAME, pull_url_updated); } return pull_url_updated; }
/** * Set some properties between the user Job and internal Job. * * @param job the user job. * @param jobToSet the internal job to set. * @throws IllegalAccessException * @throws IllegalArgumentException */ private static void setJobCommonProperties(Job job, InternalJob jobToSet) throws IllegalArgumentException, IllegalAccessException { autoCopyfields(CommonAttribute.class, job, jobToSet); autoCopyfields(Job.class, job, jobToSet); // special behavior jobToSet.setPriority(job.getPriority()); }
/** * Set some properties between the user task and internal task. * * @param task the user task. * @param taskToSet the internal task to set. * @throws IllegalAccessException * @throws IllegalArgumentException */ private static void setTaskCommonProperties(Job userJob, Task task, InternalTask taskToSet) throws IllegalArgumentException, IllegalAccessException { autoCopyfields(CommonAttribute.class, task, taskToSet); autoCopyfields(Task.class, task, taskToSet); // special behavior if (onErrorPolicyInterpreter.notSetOrNone(task)) { taskToSet.setOnTaskError(userJob.getOnTaskErrorProperty().getValue()); } else { taskToSet.setOnTaskError(task.getOnTaskErrorProperty().getValue()); } if (task.getRestartTaskOnErrorProperty().isSet()) { taskToSet.setRestartTaskOnError(task.getRestartTaskOnError()); } else { taskToSet.setRestartTaskOnError(userJob.getRestartTaskOnError()); } if (task.getMaxNumberOfExecutionProperty().isSet()) { taskToSet.setMaxNumberOfExecution(task.getMaxNumberOfExecution()); } else { taskToSet.setMaxNumberOfExecution(userJob.getMaxNumberOfExecution()); } }
/** * Does the following steps: * * <ul> * <li>Prepares the temporary folders. See {@link #prepareDataFolderAndUpdateJob(Job, String, * String, String, String)} * <li>pushes all files from localInputFolderPath to the push_url location (see {@link * #pushData(Job, String)}) * <li>submits the job to the scheduler * <li>adds the job to the awaited jobs list in order to download the output data when the job * is finished * </ul> * * Note: this method is synchronous. The caller will be blocked until the the data is pushed and * the job submitted. * * @param job job object to be submitted to the Scheduler Server for execution * @param localInputFolderPath path to the folder containing the input data for this job * @param push_url the url where input data is to be pushed before the job submission * @param localOutputFolderPath path to the folder where the output data produced by tasks in this * job should be copied * @param pull_url the url where the data is to be retrieved after the job is finished * @param localOutputFolderPath * @return * @throws JobCreationException * @throws SubmissionClosedException * @throws PermissionException * @throws NotConnectedException * @throws FileSystemException */ public JobId submit( Job job, String localInputFolderPath, String push_url, String localOutputFolderPath, String pull_url) throws NotConnectedException, PermissionException, SubmissionClosedException, JobCreationException, FileSystemException { if (((push_url == null) || (push_url.equals(""))) && ((pull_url == null) || (pull_url.equals("")))) { logger.warn( "For the job " + job.getId() + " no push or pull urls are defined. No data will be transfered for this job from the local machine "); return super.submit(job); } String newFolderName = createNewFolderName(); String push_Url_update = prepareJobInput(job, localInputFolderPath, push_url, newFolderName); String pull_url_update = prepareJobOutput(job, localOutputFolderPath, pull_url, newFolderName); pushData(job, localInputFolderPath); JobId id = super.submit(job); AwaitedJob aj = new AwaitedJob( id.toString(), localInputFolderPath, job.getInputSpace(), push_Url_update, localOutputFolderPath, job.getOutputSpace(), pull_url_update); addAwaitedJob(aj); return id; }
/** * @param job * @param inputFolder * @return * @throws FileSystemException */ protected boolean pushData(Job job, String localInputFolderPath) throws FileSystemException { String push_URL = job.getGenericInformations().get(GENERIC_INFO_PUSH_URL_PROPERTY_NAME); if ((push_URL == null) || (push_URL.trim().equals(""))) { return false; } // push inputData // TODO - if the copy fails, try to remove the files from the remote // folder before throwing an exception FileObject remoteFolder = fsManager.resolveFile(push_URL); FileObject localfolder = fsManager.resolveFile(localInputFolderPath); logger.debug("Pushing files from " + localfolder + " to " + remoteFolder); // create the selector DSFileSelector fileSelector = new DSFileSelector(); TaskFlowJob tfj = (TaskFlowJob) job; for (Task t : tfj.getTasks()) { List<InputSelector> inputFileSelectors = t.getInputFilesList(); for (InputSelector is : inputFileSelectors) { org.ow2.proactive.scheduler.common.task.dataspaces.FileSelector fs = is.getInputFiles(); if (fs.getIncludes() != null) fileSelector.addIncludes(Arrays.asList(fs.getIncludes())); if (fs.getExcludes() != null) fileSelector.addExcludes(Arrays.asList(fs.getExcludes())); } } // We need to check if a pattern exist in both includes and excludes. // This may happen if a task one defines, for instance "*.txt" as includes // and a task two defines "*.txt" as excludes. In this case we should remove it from the // fileSelector's excludes. Set<String> includes = fileSelector.getIncludes(); Set<String> excludes = fileSelector.getExcludes(); Set<String> intersection = new HashSet<String>(includes); intersection.retainAll(excludes); excludes.removeAll(intersection); fileSelector.setExcludes(excludes); remoteFolder.copyFrom(localfolder, fileSelector); logger.debug("Finished push operation from " + localfolder + " to " + remoteFolder); return true; }
/** * Tests start here. * * @throws Throwable any exception that can be thrown during the test. */ @org.junit.Test public void run() throws Throwable { String task1Name = "Task1"; String task2Name = "Task2"; String taskForked1Name = "Fork1"; String taskForked2Name = "Fork2"; TaskFlowJob job = (TaskFlowJob) JobFactory_stax.getFactory() .createJob(new File(jobDescriptor.toURI()).getAbsolutePath()); if (OperatingSystem.getOperatingSystem() == OperatingSystem.windows) { ((NativeTask) job.getTask(task1Name)) .setCommandLine("cmd", "/C", "ping", "127.0.0.1", "-n", "20", ">", "NUL"); ((NativeTask) job.getTask(taskForked1Name)) .setCommandLine("cmd", "/C", "ping", "127.0.0.1", "-n", "20", ">", "NUL"); } JobId id = SchedulerTHelper.submitJob(job); SchedulerTHelper.log("Job submitted, id " + id.toString()); SchedulerTHelper.log("Waiting for jobSubmitted Event"); Job receivedJob = SchedulerTHelper.waitForEventJobSubmitted(id); Assert.assertEquals(receivedJob.getId(), id); SchedulerTHelper.log("Waiting for job running"); JobInfo jInfo = SchedulerTHelper.waitForEventJobRunning(id); Assert.assertEquals(jInfo.getJobId(), id); Assert.assertEquals(JobStatus.RUNNING, jInfo.getStatus()); SchedulerTHelper.log("check events for task " + task1Name); TaskInfo tInfo = SchedulerTHelper.waitForEventTaskRunning(id, task1Name); Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus()); tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task1Name); Assert.assertEquals(TaskStatus.FINISHED, tInfo.getStatus()); SchedulerTHelper.log("check events for task " + task2Name); tInfo = SchedulerTHelper.waitForEventTaskRunning(id, task2Name); Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus()); tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task2Name); Assert.assertEquals(TaskStatus.FINISHED, tInfo.getStatus()); // this task reaches wall time, so finishes with faulty state SchedulerTHelper.log("check events for task " + taskForked1Name); tInfo = SchedulerTHelper.waitForEventTaskRunning(id, taskForked1Name); Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus()); tInfo = SchedulerTHelper.waitForEventTaskFinished(id, taskForked1Name); Assert.assertEquals(TaskStatus.FAULTY, tInfo.getStatus()); // this task reaches wall time, so finishes with faulty state SchedulerTHelper.log("check events for task " + taskForked2Name); tInfo = SchedulerTHelper.waitForEventTaskRunning(id, taskForked2Name); Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus()); tInfo = SchedulerTHelper.waitForEventTaskFinished(id, taskForked2Name); Assert.assertEquals(TaskStatus.FAULTY, tInfo.getStatus()); SchedulerTHelper.log("Waiting for job finished"); jInfo = SchedulerTHelper.waitForEventJobFinished(id); Assert.assertEquals(jInfo.getJobId(), id); Assert.assertEquals(JobStatus.FINISHED, jInfo.getStatus()); // check result are not null JobResult res = SchedulerTHelper.getJobResult(id); Assert.assertTrue(res.hadException()); Assert.assertFalse(res.getResult(task1Name).hadException()); Assert.assertNull(res.getResult(task1Name).getException()); Assert.assertFalse(res.getResult(task2Name).hadException()); Assert.assertNull(res.getResult(task2Name).getException()); Assert.assertFalse(res.getResult(taskForked1Name).hadException()); Assert.assertNull(res.getResult(taskForked1Name).getException()); Assert.assertTrue(res.getResult(taskForked2Name).hadException()); Assert.assertNotNull(res.getResult(taskForked2Name).getException()); // remove job SchedulerTHelper.removeJob(id); SchedulerTHelper.waitForEventJobRemoved(id); }