/**
* Tests start here.
*
* @throws Throwable any exception that can be thrown during the test.
*/
@org.junit.Test
public void run() throws Throwable {
String task1Name = "task1";
String task2Name = "task2";
// test submission and event reception
TaskFlowJob job =
(TaskFlowJob)
JobFactory_stax.getFactory()
.createJob(new File(jobDescriptor.toURI()).getAbsolutePath());
if (OperatingSystem.getOperatingSystem() == OperatingSystem.windows) {
((NativeTask) job.getTask(task1Name))
.setCommandLine("cmd", "/C", "ping", "127.0.0.1", "-n", "10", ">", "NUL");
}
JobId id = SchedulerTHelper.submitJob(job);
SchedulerTHelper.log("Job submitted, id " + id.toString());
SchedulerTHelper.log("Waiting for jobSubmitted Event");
JobState receivedState = SchedulerTHelper.waitForEventJobSubmitted(id);
Assert.assertEquals(receivedState.getId(), id);
SchedulerTHelper.log("Waiting for job running");
JobInfo jInfo = SchedulerTHelper.waitForEventJobRunning(id);
Assert.assertEquals(jInfo.getJobId(), id);
Assert.assertEquals(JobStatus.RUNNING, jInfo.getStatus());
SchedulerTHelper.waitForEventTaskRunning(id, task1Name);
TaskInfo tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task1Name);
Assert.assertEquals(TaskStatus.FINISHED, tInfo.getStatus());
SchedulerTHelper.waitForEventTaskRunning(id, task2Name);
tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task2Name);
Assert.assertEquals(TaskStatus.FAULTY, tInfo.getStatus());
SchedulerTHelper.waitForEventJobFinished(id);
JobResult res = SchedulerTHelper.getJobResult(id);
// check that there is one exception in results
Assert.assertTrue(res.getExceptionResults().size() == 1);
// remove job
SchedulerTHelper.removeJob(id);
SchedulerTHelper.waitForEventJobRemoved(id);
}
/**
* @param job
* @param inputFolder
* @return
* @throws FileSystemException
*/
protected boolean pushData(Job job, String localInputFolderPath) throws FileSystemException {
String push_URL = job.getGenericInformations().get(GENERIC_INFO_PUSH_URL_PROPERTY_NAME);
if ((push_URL == null) || (push_URL.trim().equals(""))) {
return false;
} // push inputData
// TODO - if the copy fails, try to remove the files from the remote
// folder before throwing an exception
FileObject remoteFolder = fsManager.resolveFile(push_URL);
FileObject localfolder = fsManager.resolveFile(localInputFolderPath);
logger.debug("Pushing files from " + localfolder + " to " + remoteFolder);
// create the selector
DSFileSelector fileSelector = new DSFileSelector();
TaskFlowJob tfj = (TaskFlowJob) job;
for (Task t : tfj.getTasks()) {
List<InputSelector> inputFileSelectors = t.getInputFilesList();
for (InputSelector is : inputFileSelectors) {
org.ow2.proactive.scheduler.common.task.dataspaces.FileSelector fs = is.getInputFiles();
if (fs.getIncludes() != null) fileSelector.addIncludes(Arrays.asList(fs.getIncludes()));
if (fs.getExcludes() != null) fileSelector.addExcludes(Arrays.asList(fs.getExcludes()));
}
}
// We need to check if a pattern exist in both includes and excludes.
// This may happen if a task one defines, for instance "*.txt" as includes
// and a task two defines "*.txt" as excludes. In this case we should remove it from the
// fileSelector's excludes.
Set<String> includes = fileSelector.getIncludes();
Set<String> excludes = fileSelector.getExcludes();
Set<String> intersection = new HashSet<String>(includes);
intersection.retainAll(excludes);
excludes.removeAll(intersection);
fileSelector.setExcludes(excludes);
remoteFolder.copyFrom(localfolder, fileSelector);
logger.debug("Finished push operation from " + localfolder + " to " + remoteFolder);
return true;
}
/** Creates the taskflow element, corresponding to <define name="taskFlow"> */
private Element createTaskFlowElement(Document doc, TaskFlowJob job) {
Element taskFlowElement =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK_FLOW.getXMLName());
ArrayList<Task> tasks = job.getTasks();
// <oneOrMore>
// <ref name="task"/>
// </oneOrMore>
for (Task task : tasks) {
Element taskE = createTaskElement(doc, task);
taskFlowElement.appendChild(taskE);
}
return taskFlowElement;
}
/**
* Tests start here.
*
* @throws Throwable any exception that can be thrown during the test.
*/
@org.junit.Test
public void run() throws Throwable {
String task1Name = "Task1";
String task2Name = "Task2";
String taskForked1Name = "Fork1";
String taskForked2Name = "Fork2";
TaskFlowJob job =
(TaskFlowJob)
JobFactory_stax.getFactory()
.createJob(new File(jobDescriptor.toURI()).getAbsolutePath());
if (OperatingSystem.getOperatingSystem() == OperatingSystem.windows) {
((NativeTask) job.getTask(task1Name))
.setCommandLine("cmd", "/C", "ping", "127.0.0.1", "-n", "20", ">", "NUL");
((NativeTask) job.getTask(taskForked1Name))
.setCommandLine("cmd", "/C", "ping", "127.0.0.1", "-n", "20", ">", "NUL");
}
JobId id = SchedulerTHelper.submitJob(job);
SchedulerTHelper.log("Job submitted, id " + id.toString());
SchedulerTHelper.log("Waiting for jobSubmitted Event");
Job receivedJob = SchedulerTHelper.waitForEventJobSubmitted(id);
Assert.assertEquals(receivedJob.getId(), id);
SchedulerTHelper.log("Waiting for job running");
JobInfo jInfo = SchedulerTHelper.waitForEventJobRunning(id);
Assert.assertEquals(jInfo.getJobId(), id);
Assert.assertEquals(JobStatus.RUNNING, jInfo.getStatus());
SchedulerTHelper.log("check events for task " + task1Name);
TaskInfo tInfo = SchedulerTHelper.waitForEventTaskRunning(id, task1Name);
Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus());
tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task1Name);
Assert.assertEquals(TaskStatus.FINISHED, tInfo.getStatus());
SchedulerTHelper.log("check events for task " + task2Name);
tInfo = SchedulerTHelper.waitForEventTaskRunning(id, task2Name);
Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus());
tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task2Name);
Assert.assertEquals(TaskStatus.FINISHED, tInfo.getStatus());
// this task reaches wall time, so finishes with faulty state
SchedulerTHelper.log("check events for task " + taskForked1Name);
tInfo = SchedulerTHelper.waitForEventTaskRunning(id, taskForked1Name);
Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus());
tInfo = SchedulerTHelper.waitForEventTaskFinished(id, taskForked1Name);
Assert.assertEquals(TaskStatus.FAULTY, tInfo.getStatus());
// this task reaches wall time, so finishes with faulty state
SchedulerTHelper.log("check events for task " + taskForked2Name);
tInfo = SchedulerTHelper.waitForEventTaskRunning(id, taskForked2Name);
Assert.assertEquals(TaskStatus.RUNNING, tInfo.getStatus());
tInfo = SchedulerTHelper.waitForEventTaskFinished(id, taskForked2Name);
Assert.assertEquals(TaskStatus.FAULTY, tInfo.getStatus());
SchedulerTHelper.log("Waiting for job finished");
jInfo = SchedulerTHelper.waitForEventJobFinished(id);
Assert.assertEquals(jInfo.getJobId(), id);
Assert.assertEquals(JobStatus.FINISHED, jInfo.getStatus());
// check result are not null
JobResult res = SchedulerTHelper.getJobResult(id);
Assert.assertTrue(res.hadException());
Assert.assertFalse(res.getResult(task1Name).hadException());
Assert.assertNull(res.getResult(task1Name).getException());
Assert.assertFalse(res.getResult(task2Name).hadException());
Assert.assertNull(res.getResult(task2Name).getException());
Assert.assertFalse(res.getResult(taskForked1Name).hadException());
Assert.assertNull(res.getResult(taskForked1Name).getException());
Assert.assertTrue(res.getResult(taskForked2Name).hadException());
Assert.assertNotNull(res.getResult(taskForked2Name).getException());
// remove job
SchedulerTHelper.removeJob(id);
SchedulerTHelper.waitForEventJobRemoved(id);
}
/**
* Create an internalTaskFlow job with the given task flow job (user)
*
* @param userJob the user job that will be used to create the internal job.
* @return the created internal job.
* @throws JobCreationException an exception if the factory cannot create the given job.
*/
private static InternalJob createJob(TaskFlowJob userJob) throws JobCreationException {
if (userJob.getTasks().size() == 0) {
logger.info("Job '" + userJob.getName() + "' must contain tasks !");
throw new JobCreationException("This job must contains tasks !");
}
// validate taskflow
List<FlowChecker.Block> blocks = new ArrayList<>();
FlowError err = FlowChecker.validate(userJob, blocks);
if (err != null) {
String e = "";
e += "Invalid taskflow: " + err.getMessage() + "; context: " + err.getTask();
logger.error(e);
throw new JobCreationException(e, err);
}
InternalJob job = new InternalTaskFlowJob();
// keep an initial job content
job.setTaskFlowJob(userJob);
Map<Task, InternalTask> tasksList = new LinkedHashMap<>();
boolean hasPreciousResult = false;
for (Task t : userJob.getTasks()) {
tasksList.put(t, createTask(userJob, t));
if (!hasPreciousResult) {
hasPreciousResult = t.isPreciousResult();
}
}
for (Entry<Task, InternalTask> entry : tasksList.entrySet()) {
if (entry.getKey().getDependencesList() != null) {
for (Task t : entry.getKey().getDependencesList()) {
entry.getValue().addDependence(tasksList.get(t));
}
}
job.addTask(entry.getValue());
}
// tag matching blocks in InternalTasks
for (InternalTask it : tasksList.values()) {
for (FlowChecker.Block block : blocks) {
if (it.getName().equals(block.start.element.getName())) {
it.setMatchingBlock(block.end.element.getName());
}
if (it.getName().equals(block.end.element.getName())) {
it.setMatchingBlock(block.start.element.getName());
}
}
}
// create if/else/join weak dependencies
for (InternalTask it : tasksList.values()) {
// it performs an IF action
if (it.getFlowScript() != null
&& it.getFlowScript().getActionType().equals(FlowActionType.IF.toString())) {
String ifBranch = it.getFlowScript().getActionTarget();
String elseBranch = it.getFlowScript().getActionTargetElse();
String join = it.getFlowScript().getActionContinuation();
List<InternalTask> joinedBranches = new ArrayList<>();
// find the ifBranch task
for (InternalTask it2 : tasksList.values()) {
if (it2.getName().equals(ifBranch)) {
it2.setIfBranch(it);
String match = it2.getMatchingBlock();
// find its matching block task
if (match == null) {
// no match: single task
joinedBranches.add(it2);
} else {
for (InternalTask it3 : tasksList.values()) {
if (it3.getName().equals(match)) {
joinedBranches.add(it3);
break;
}
}
}
break;
}
}
// find the elseBranch task
for (InternalTask it2 : tasksList.values()) {
if (it2.getName().equals(elseBranch)) {
it2.setIfBranch(it);
String match = it2.getMatchingBlock();
// find its matching block task
if (match == null) {
// no match: single task
joinedBranches.add(it2);
} else {
for (InternalTask it3 : tasksList.values()) {
if (it3.getName().equals(match)) {
joinedBranches.add(it3);
break;
}
}
}
break;
}
}
// find the joinBranch task
for (InternalTask it2 : tasksList.values()) {
if (it2.getName().equals(join)) {
it2.setJoinedBranches(joinedBranches);
}
}
}
}
return job;
}
/** Creates the "job" element <define name="job"> */
private Element createRootJobElement(Document doc, TaskFlowJob job) {
Element rootJob = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, "job");
// ********** attributes ***********
rootJob.setAttributeNS(
"http://www.w3.org/2001/XMLSchema-instance", "xsi:schemaLocation", XSD_LOCATION);
setAttribute(rootJob, XMLAttributes.JOB_PROJECT_NAME, job.getProjectName(), true);
setAttribute(rootJob, XMLAttributes.JOB_PRIORITY, job.getPriority().toString());
if (job.getOnTaskErrorProperty().isSet()) {
setAttribute(
rootJob,
XMLAttributes.COMMON_ON_TASK_ERROR,
job.getOnTaskErrorProperty().getValue().toString(),
true);
}
if (job.getMaxNumberOfExecutionProperty().isSet()) {
setAttribute(
rootJob,
XMLAttributes.COMMON_MAX_NUMBER_OF_EXECUTION,
Integer.toString(job.getMaxNumberOfExecution()));
}
setAttribute(rootJob, XMLAttributes.COMMON_NAME, job.getName(), true);
if (job.getRestartTaskOnErrorProperty().isSet()) {
setAttribute(
rootJob,
XMLAttributes.COMMON_RESTART_TASK_ON_ERROR,
job.getRestartTaskOnError().toString());
}
// *** elements ***
// <ref name="variables"/>
if (job.getVariables() != null && !job.getVariables().isEmpty()) {
Element variablesE = createVariablesElement(doc, job.getVariables());
rootJob.appendChild(variablesE);
}
// <ref name="jobDescription"/>
if (job.getDescription() != null) {
Element descrNode =
createElement(doc, XMLTags.COMMON_DESCRIPTION.getXMLName(), job.getDescription());
rootJob.appendChild(descrNode);
}
// <ref name="genericInformation"/>
if ((job.getGenericInformation() != null) && (job.getGenericInformation().size() > 0)) {
Element genericInfo = createGenericInformation(doc, job.getGenericInformation());
rootJob.appendChild(genericInfo);
}
// <ref name="inputSpace"/>
if (job.getInputSpace() != null) {
Element inputspace =
createElement(
doc,
XMLTags.DS_INPUT_SPACE.getXMLName(),
null,
new Attribute(XMLAttributes.DS_URL.getXMLName(), job.getInputSpace()));
rootJob.appendChild(inputspace);
}
// <ref name="outputSpace"/>
if (job.getOutputSpace() != null) {
Element outputSpace =
createElement(
doc,
XMLTags.DS_OUTPUT_SPACE.getXMLName(),
null,
new Attribute(XMLAttributes.DS_URL.getXMLName(), job.getOutputSpace()));
rootJob.appendChild(outputSpace);
}
// <ref name="globalSpace"/>
if (job.getGlobalSpace() != null) {
Element globalSpace =
createElement(
doc,
XMLTags.DS_GLOBAL_SPACE.getXMLName(),
null,
new Attribute(XMLAttributes.DS_URL.getXMLName(), job.getGlobalSpace()));
rootJob.appendChild(globalSpace);
}
// <ref name="userSpace"/>
if (job.getUserSpace() != null) {
Element userSpace =
createElement(
doc,
XMLTags.DS_USER_SPACE.getXMLName(),
null,
new Attribute(XMLAttributes.DS_URL.getXMLName(), job.getUserSpace()));
rootJob.appendChild(userSpace);
}
// <ref name="taskFlow"/>
Element taskFlow = createTaskFlowElement(doc, job);
rootJob.appendChild(taskFlow);
return rootJob;
}
