/** * @param job * @param inputFolder * @return * @throws FileSystemException */ protected boolean pushData(Job job, String localInputFolderPath) throws FileSystemException { String push_URL = job.getGenericInformations().get(GENERIC_INFO_PUSH_URL_PROPERTY_NAME); if ((push_URL == null) || (push_URL.trim().equals(""))) { return false; } // push inputData // TODO - if the copy fails, try to remove the files from the remote // folder before throwing an exception FileObject remoteFolder = fsManager.resolveFile(push_URL); FileObject localfolder = fsManager.resolveFile(localInputFolderPath); logger.debug("Pushing files from " + localfolder + " to " + remoteFolder); // create the selector DSFileSelector fileSelector = new DSFileSelector(); TaskFlowJob tfj = (TaskFlowJob) job; for (Task t : tfj.getTasks()) { List<InputSelector> inputFileSelectors = t.getInputFilesList(); for (InputSelector is : inputFileSelectors) { org.ow2.proactive.scheduler.common.task.dataspaces.FileSelector fs = is.getInputFiles(); if (fs.getIncludes() != null) fileSelector.addIncludes(Arrays.asList(fs.getIncludes())); if (fs.getExcludes() != null) fileSelector.addExcludes(Arrays.asList(fs.getExcludes())); } } // We need to check if a pattern exist in both includes and excludes. // This may happen if a task one defines, for instance "*.txt" as includes // and a task two defines "*.txt" as excludes. In this case we should remove it from the // fileSelector's excludes. Set<String> includes = fileSelector.getIncludes(); Set<String> excludes = fileSelector.getExcludes(); Set<String> intersection = new HashSet<String>(includes); intersection.retainAll(excludes); excludes.removeAll(intersection); fileSelector.setExcludes(excludes); remoteFolder.copyFrom(localfolder, fileSelector); logger.debug("Finished push operation from " + localfolder + " to " + remoteFolder); return true; }
/** * Set some properties between the user task and internal task. * * @param task the user task. * @param taskToSet the internal task to set. * @throws IllegalAccessException * @throws IllegalArgumentException */ private static void setTaskCommonProperties(Job userJob, Task task, InternalTask taskToSet) throws IllegalArgumentException, IllegalAccessException { autoCopyfields(CommonAttribute.class, task, taskToSet); autoCopyfields(Task.class, task, taskToSet); // special behavior if (onErrorPolicyInterpreter.notSetOrNone(task)) { taskToSet.setOnTaskError(userJob.getOnTaskErrorProperty().getValue()); } else { taskToSet.setOnTaskError(task.getOnTaskErrorProperty().getValue()); } if (task.getRestartTaskOnErrorProperty().isSet()) { taskToSet.setRestartTaskOnError(task.getRestartTaskOnError()); } else { taskToSet.setRestartTaskOnError(userJob.getRestartTaskOnError()); } if (task.getMaxNumberOfExecutionProperty().isSet()) { taskToSet.setMaxNumberOfExecution(task.getMaxNumberOfExecution()); } else { taskToSet.setMaxNumberOfExecution(userJob.getMaxNumberOfExecution()); } }
private static InternalTask createTask(Job userJob, Task task) throws JobCreationException { // TODO: avoid branching with double dispatch if (task instanceof NativeTask) { return createTask(userJob, (NativeTask) task); } else if (task instanceof JavaTask) { return createTask(userJob, (JavaTask) task); } else if (task instanceof ScriptTask) { return createTask(userJob, (ScriptTask) task); } String msg = "Unknown task type: " + task.getClass().getName(); logger.info(msg); throw new JobCreationException(msg); }
private static void configureRunAsMe(Task task) { if (isRunAsMeTask()) { task.setRunAsMe(true); } }
/** * Create an internalTaskFlow job with the given task flow job (user) * * @param userJob the user job that will be used to create the internal job. * @return the created internal job. * @throws JobCreationException an exception if the factory cannot create the given job. */ private static InternalJob createJob(TaskFlowJob userJob) throws JobCreationException { if (userJob.getTasks().size() == 0) { logger.info("Job '" + userJob.getName() + "' must contain tasks !"); throw new JobCreationException("This job must contains tasks !"); } // validate taskflow List<FlowChecker.Block> blocks = new ArrayList<>(); FlowError err = FlowChecker.validate(userJob, blocks); if (err != null) { String e = ""; e += "Invalid taskflow: " + err.getMessage() + "; context: " + err.getTask(); logger.error(e); throw new JobCreationException(e, err); } InternalJob job = new InternalTaskFlowJob(); // keep an initial job content job.setTaskFlowJob(userJob); Map<Task, InternalTask> tasksList = new LinkedHashMap<>(); boolean hasPreciousResult = false; for (Task t : userJob.getTasks()) { tasksList.put(t, createTask(userJob, t)); if (!hasPreciousResult) { hasPreciousResult = t.isPreciousResult(); } } for (Entry<Task, InternalTask> entry : tasksList.entrySet()) { if (entry.getKey().getDependencesList() != null) { for (Task t : entry.getKey().getDependencesList()) { entry.getValue().addDependence(tasksList.get(t)); } } job.addTask(entry.getValue()); } // tag matching blocks in InternalTasks for (InternalTask it : tasksList.values()) { for (FlowChecker.Block block : blocks) { if (it.getName().equals(block.start.element.getName())) { it.setMatchingBlock(block.end.element.getName()); } if (it.getName().equals(block.end.element.getName())) { it.setMatchingBlock(block.start.element.getName()); } } } // create if/else/join weak dependencies for (InternalTask it : tasksList.values()) { // it performs an IF action if (it.getFlowScript() != null && it.getFlowScript().getActionType().equals(FlowActionType.IF.toString())) { String ifBranch = it.getFlowScript().getActionTarget(); String elseBranch = it.getFlowScript().getActionTargetElse(); String join = it.getFlowScript().getActionContinuation(); List<InternalTask> joinedBranches = new ArrayList<>(); // find the ifBranch task for (InternalTask it2 : tasksList.values()) { if (it2.getName().equals(ifBranch)) { it2.setIfBranch(it); String match = it2.getMatchingBlock(); // find its matching block task if (match == null) { // no match: single task joinedBranches.add(it2); } else { for (InternalTask it3 : tasksList.values()) { if (it3.getName().equals(match)) { joinedBranches.add(it3); break; } } } break; } } // find the elseBranch task for (InternalTask it2 : tasksList.values()) { if (it2.getName().equals(elseBranch)) { it2.setIfBranch(it); String match = it2.getMatchingBlock(); // find its matching block task if (match == null) { // no match: single task joinedBranches.add(it2); } else { for (InternalTask it3 : tasksList.values()) { if (it3.getName().equals(match)) { joinedBranches.add(it3); break; } } } break; } } // find the joinBranch task for (InternalTask it2 : tasksList.values()) { if (it2.getName().equals(join)) { it2.setJoinedBranches(joinedBranches); } } } } return job; }
/** * Creates a flow control element for the given task <element name="controlFlow"> * * @return the xml Element corresponding to the flow control, if the task contains a flow control, * null otherwise */ private Element createFlowControlElement(Document doc, Task task) { Element controlFlowE = null; // <ref name="block"/> if (task.getFlowBlock() != FlowBlock.NONE) { controlFlowE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW.getXMLName()); setAttribute(controlFlowE, XMLAttributes.FLOW_BLOCK, task.getFlowBlock().toString()); } FlowScript flowScript = task.getFlowScript(); if (flowScript != null) { Element flowActionE = null; // flowActionE can be if, loop, replicate or null. // if not null, it contains a script element // <ref name="actionIf"/> // <ref name="actionReplicate"/> // <ref name="actionLoop"/> // </choice> // *** if *** // <element name="if"> if (flowScript.getActionType().equals(FlowActionType.IF.toString())) { flowActionE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW_IF.getXMLName()); setAttribute(flowActionE, XMLAttributes.FLOW_TARGET, flowScript.getActionTarget(), true); setAttribute(flowActionE, XMLAttributes.FLOW_ELSE, flowScript.getActionTargetElse(), true); setAttribute( flowActionE, XMLAttributes.FLOW_CONTINUATION, flowScript.getActionContinuation(), true); } // *** loop *** // <element name="loop"> if (flowScript.getActionType().equals(FlowActionType.LOOP.toString())) { flowActionE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW_LOOP.getXMLName()); setAttribute(flowActionE, XMLAttributes.FLOW_TARGET, flowScript.getActionTarget(), true); } // *** replicate *** // <element name="replicate"> if (flowScript.getActionType().equals(FlowActionType.REPLICATE.toString())) { flowActionE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW_REPLICATE.getXMLName()); } if (flowActionE != null) { if (controlFlowE == null) { controlFlowE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW.getXMLName()); } Element scriptE = createScriptElement(doc, flowScript); flowActionE.appendChild(scriptE); controlFlowE.appendChild(flowActionE); } } // flowScript !=null return controlFlowE; }
/** * Creates the parallel environment element for the given task. Corresponds to <define * name="parallel"> * * @return the {@link XMLTags#PARALLEL_ENV} element if the task has a parallel environment, null * otherwise */ private Element createParallelEnvironment(Document doc, Task task) { ParallelEnvironment penv = task.getParallelEnvironment(); if (penv == null) return null; Element parallelEnvE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.PARALLEL_ENV.getXMLName()); setAttribute( parallelEnvE, XMLAttributes.TASK_NB_NODES, Integer.toString(penv.getNodesNumber())); // <ref name="topology"/> TopologyDescriptor topologyDescr = penv.getTopologyDescriptor(); if (topologyDescr != null) { // <choice> // <ref name="arbitrary"/> // <ref name="bestProximity"/> // <ref name="thresholdProximity"/> // <ref name="singleHost"/> // <ref name="singleHostExclusive"/> // <ref name="multipleHostsExclusive"/> // <ref name="differentHostsExclusive"/> // </choice> Element topologyE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY.getXMLName()); Element topologyDescrE = null; if (topologyDescr instanceof ArbitraryTopologyDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_ARBITRARY.getXMLName()); } else if (topologyDescr instanceof ThresholdProximityDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_THRESHOLD_PROXIMITY.getXMLName()); long threshold = ((ThresholdProximityDescriptor) topologyDescr).getThreshold(); topologyDescrE.setAttribute( XMLAttributes.TOPOLOGY_THRESHOLD.getXMLName(), Long.toString(threshold)); } else if (topologyDescr instanceof BestProximityDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_BEST_PROXIMITY.getXMLName()); } else if (topologyDescr instanceof SingleHostExclusiveDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_SINGLE_HOST_EXCLUSIVE.getXMLName()); } else if (topologyDescr instanceof SingleHostDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_SINGLE_HOST.getXMLName()); } else if (topologyDescr instanceof MultipleHostsExclusiveDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_MULTIPLE_HOSTS_EXCLUSIVE.getXMLName()); } if (topologyDescr instanceof DifferentHostsExclusiveDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_DIFFERENT_HOSTS_EXCLUSIVE.getXMLName()); } if (topologyDescrE != null) { topologyE.appendChild(topologyDescrE); } parallelEnvE.appendChild(topologyE); } return parallelEnvE; }
/** Creates the task element, corressponding to <define name="task"> */ private Element createTaskElement(Document doc, Task task) { Element taskE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK.getXMLName()); // **** attributes ***** // **** common attributes *** if (task.getOnTaskErrorProperty().isSet()) { setAttribute( taskE, XMLAttributes.COMMON_ON_TASK_ERROR, task.getOnTaskErrorProperty().getValue().toString(), true); } if (task.getMaxNumberOfExecutionProperty().isSet()) { setAttribute( taskE, XMLAttributes.COMMON_MAX_NUMBER_OF_EXECUTION, Integer.toString(task.getMaxNumberOfExecution())); } setAttribute(taskE, XMLAttributes.COMMON_NAME, task.getName(), true); if (task.getRestartTaskOnErrorProperty().isSet()) { setAttribute( taskE, XMLAttributes.COMMON_RESTART_TASK_ON_ERROR, task.getRestartTaskOnError().toString()); } // *** task attributes *** if (task.getWallTime() != 0) { setAttribute(taskE, XMLAttributes.TASK_WALLTIME, formatDate(task.getWallTime())); } if (task.isRunAsMe()) { setAttribute(taskE, XMLAttributes.TASK_RUN_AS_ME, "true"); } if (task.isPreciousResult()) { setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_RESULT, "true"); } if (task.isPreciousLogs()) { setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_LOGS, "true"); } // *** elements **** // <ref name="variables"/> if (task.getVariables() != null && !task.getVariables().isEmpty()) { Element variablesE = createTaskVariablesElement(doc, task.getVariables()); taskE.appendChild(variablesE); } // <ref name="taskDescription"/> if (task.getDescription() != null) { Element descrNode = createElement(doc, XMLTags.COMMON_DESCRIPTION.getXMLName(), task.getDescription()); taskE.appendChild(descrNode); } // <ref name="genericInformation"/> if ((task.getGenericInformation() != null) && (task.getGenericInformation().size() > 0)) { Element genericInfoE = createGenericInformation(doc, task.getGenericInformation()); taskE.appendChild(genericInfoE); } // <ref name="depends"/> List<Task> dependencies = task.getDependencesList(); if ((dependencies != null) && (dependencies.size() > 0)) { Element dependsE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK_DEPENDENCES.getXMLName()); for (Task dep : dependencies) { Element dependsTask = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK_DEPENDENCES_TASK.getXMLName()); setAttribute(dependsTask, XMLAttributes.TASK_DEPENDS_REF, dep.getName(), true); dependsE.appendChild(dependsTask); } taskE.appendChild(dependsE); } // if has dependencies // <ref name="inputFiles"/> List<InputSelector> inputFiles = task.getInputFilesList(); if (inputFiles != null) { Element inputFilesE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_INPUT_FILES.getXMLName()); for (InputSelector inputSelector : inputFiles) { FileSelector fs = inputSelector.getInputFiles(); Element filesE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_FILES.getXMLName()); // the xml only supports one value for the includes/excludes // pattern if (!fs.getIncludes().isEmpty()) setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true); if (!fs.getExcludes().isEmpty()) setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true); if (inputSelector.getMode() != null) { setAttribute( filesE, XMLAttributes.DS_ACCESS_MODE, inputSelector.getMode().toString(), true); } inputFilesE.appendChild(filesE); } taskE.appendChild(inputFilesE); } // <ref name="parallel"/> Element parallelEnvE = createParallelEnvironment(doc, task); if (parallelEnvE != null) taskE.appendChild(parallelEnvE); // <ref name="selection"/> List<SelectionScript> selectionScripts = task.getSelectionScripts(); if (selectionScripts != null && selectionScripts.size() > 0) { Element selectionE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_SELECTION.getXMLName()); for (SelectionScript selectionScript : selectionScripts) { Element scriptE = createScriptElement(doc, selectionScript); selectionE.appendChild(scriptE); } taskE.appendChild(selectionE); } // <ref name="forkEnvironment"/> if (task.getForkEnvironment() != null) { Element forkEnvE = createForkEnvironmentElement(doc, task.getForkEnvironment()); taskE.appendChild(forkEnvE); } // <ref name="pre"/> Script preScript = task.getPreScript(); if (preScript != null) { Element preE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_PRE.getXMLName()); Element scriptE = createScriptElement(doc, preScript); preE.appendChild(scriptE); taskE.appendChild(preE); } // <ref name="executable"/> Element executableE = null; if (task instanceof JavaTask) { executableE = createJavaExecutableElement(doc, (JavaTask) task); } else if (task instanceof NativeTask) { executableE = createNativeExecutableElement(doc, (NativeTask) task); } else if (task instanceof ScriptTask) { executableE = createScriptExecutableElement(doc, (ScriptTask) task); } taskE.appendChild(executableE); // <ref name="flow"/> Element controlFlowE = createFlowControlElement(doc, task); if (controlFlowE != null) taskE.appendChild(controlFlowE); // <ref name="post"/> Script postScript = task.getPostScript(); if (postScript != null) { Element postE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_POST.getXMLName()); Element scriptE = createScriptElement(doc, postScript); postE.appendChild(scriptE); taskE.appendChild(postE); } // <ref name="cleaning"/> Script cleanScript = task.getCleaningScript(); if (cleanScript != null) { Element cleanE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_CLEANING.getXMLName()); Element scriptE = createScriptElement(doc, cleanScript); cleanE.appendChild(scriptE); taskE.appendChild(cleanE); } // <ref name="outputFiles"/> List<OutputSelector> outputFiles = task.getOutputFilesList(); if (outputFiles != null) { Element outputFilesE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_OUTPUT_FILES.getXMLName()); for (OutputSelector outputSelector : outputFiles) { FileSelector fs = outputSelector.getOutputFiles(); Element filesE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_FILES.getXMLName()); // the xml only supports one value for the includes/excludes // pattern if (!fs.getIncludes().isEmpty()) setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true); if (!fs.getExcludes().isEmpty()) setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true); if (outputSelector.getMode() != null) { setAttribute( filesE, XMLAttributes.DS_ACCESS_MODE, outputSelector.getMode().toString(), true); } outputFilesE.appendChild(filesE); } taskE.appendChild(outputFilesE); } return taskE; }
/** * Creates and submit a job from an XML job descriptor, and check, with assertions, event related * to this job submission : 1/ job submitted event 2/ job passing from pending to running (with * state set to running). 3/ every task passing from pending to running (with state set to * running). 4/ every task finish without error ; passing from running to finished (with state set * to finished). 5/ and finally job passing from running to finished (with state set to finished). * Then returns. * * <p>This is the simplest events sequence of a job submission. If you need to test specific * events or task states (failures, rescheduling etc, you must not use this helper and check * events sequence with waitForEvent**() functions. * * @param jobToSubmit job object to schedule. * @return JobId, the job's identifier. * @throws Exception if an error occurs at job submission, or during verification of events * sequence. */ public JobId testJobSubmission(Job jobToSubmit) throws Exception { Scheduler userInt = getSchedulerInterface(); JobId id = userInt.submit(jobToSubmit); log("Job submitted, id " + id.toString()); log("Waiting for jobSubmitted"); JobState receivedState = waitForEventJobSubmitted(id); Assert.assertEquals(id, receivedState.getId()); log("Waiting for job running"); JobInfo jInfo = waitForEventJobRunning(id); Assert.assertEquals(jInfo.getJobId(), id); Assert.assertEquals("Job " + jInfo.getJobId(), JobStatus.RUNNING, jInfo.getStatus()); if (jobToSubmit instanceof TaskFlowJob) { for (Task t : ((TaskFlowJob) jobToSubmit).getTasks()) { log("Waiting for task running : " + t.getName()); TaskInfo ti = waitForEventTaskRunning(id, t.getName()); Assert.assertEquals(t.getName(), ti.getTaskId().getReadableName()); Assert.assertEquals("Task " + t.getName(), TaskStatus.RUNNING, ti.getStatus()); } for (Task t : ((TaskFlowJob) jobToSubmit).getTasks()) { log("Waiting for task finished : " + t.getName()); TaskInfo ti = waitForEventTaskFinished(id, t.getName()); Assert.assertEquals(t.getName(), ti.getTaskId().getReadableName()); if (ti.getStatus() == TaskStatus.FAULTY) { TaskResult tres = userInt.getTaskResult(jInfo.getJobId(), t.getName()); Assert.assertNotNull("Task result of " + t.getName(), tres); if (tres.getOutput() != null) { System.err.println("Output of failing task (" + t.getName() + ") :"); System.err.println(tres.getOutput().getAllLogs(true)); } if (tres.hadException()) { System.err.println("Exception occurred in task (" + t.getName() + ") :"); tres.getException().printStackTrace(System.err); } } Assert.assertEquals("Task " + t.getName(), TaskStatus.FINISHED, ti.getStatus()); } } log("Waiting for job finished"); jInfo = waitForEventJobFinished(id); Assert.assertEquals("Job " + jInfo.getJobId(), JobStatus.FINISHED, jInfo.getStatus()); log("Job finished"); return id; }