public void checkAndSend() {
try {
boolean sent = doCheckAndSend();
if (sent) {
jlogger.info(jobState.getId(), "sent notification email for finished job");
}
} catch (JobEmailNotificationException e) {
jlogger.warn(jobState.getId(), "failed to send email notification: " + e.getMessage());
logger.trace("Stack trace:", e);
}
}
/**
* Creates and submit a job from an XML job descriptor, and check, with assertions, event related
* to this job submission : 1/ job submitted event 2/ job passing from pending to running (with
* state set to running). 3/ every task passing from pending to running (with state set to
* running). 4/ every task finish without error ; passing from running to finished (with state set
* to finished). 5/ and finally job passing from running to finished (with state set to finished).
* Then returns.
*
* <p>This is the simplest events sequence of a job submission. If you need to test specific
* events or task states (failures, rescheduling etc, you must not use this helper and check
* events sequence with waitForEvent**() functions.
*
* @param jobToSubmit job object to schedule.
* @return JobId, the job's identifier.
* @throws Exception if an error occurs at job submission, or during verification of events
* sequence.
*/
public JobId testJobSubmission(Job jobToSubmit) throws Exception {
Scheduler userInt = getSchedulerInterface();
JobId id = userInt.submit(jobToSubmit);
log("Job submitted, id " + id.toString());
log("Waiting for jobSubmitted");
JobState receivedState = waitForEventJobSubmitted(id);
Assert.assertEquals(id, receivedState.getId());
log("Waiting for job running");
JobInfo jInfo = waitForEventJobRunning(id);
Assert.assertEquals(jInfo.getJobId(), id);
Assert.assertEquals("Job " + jInfo.getJobId(), JobStatus.RUNNING, jInfo.getStatus());
if (jobToSubmit instanceof TaskFlowJob) {
for (Task t : ((TaskFlowJob) jobToSubmit).getTasks()) {
log("Waiting for task running : " + t.getName());
TaskInfo ti = waitForEventTaskRunning(id, t.getName());
Assert.assertEquals(t.getName(), ti.getTaskId().getReadableName());
Assert.assertEquals("Task " + t.getName(), TaskStatus.RUNNING, ti.getStatus());
}
for (Task t : ((TaskFlowJob) jobToSubmit).getTasks()) {
log("Waiting for task finished : " + t.getName());
TaskInfo ti = waitForEventTaskFinished(id, t.getName());
Assert.assertEquals(t.getName(), ti.getTaskId().getReadableName());
if (ti.getStatus() == TaskStatus.FAULTY) {
TaskResult tres = userInt.getTaskResult(jInfo.getJobId(), t.getName());
Assert.assertNotNull("Task result of " + t.getName(), tres);
if (tres.getOutput() != null) {
System.err.println("Output of failing task (" + t.getName() + ") :");
System.err.println(tres.getOutput().getAllLogs(true));
}
if (tres.hadException()) {
System.err.println("Exception occurred in task (" + t.getName() + ") :");
tres.getException().printStackTrace(System.err);
}
}
Assert.assertEquals("Task " + t.getName(), TaskStatus.FINISHED, ti.getStatus());
}
}
log("Waiting for job finished");
jInfo = waitForEventJobFinished(id);
Assert.assertEquals("Job " + jInfo.getJobId(), JobStatus.FINISHED, jInfo.getStatus());
log("Job finished");
return id;
}
/**
* Tests start here.
*
* @throws Throwable any exception that can be thrown during the test.
*/
@org.junit.Test
public void run() throws Throwable {
String task1Name = "task1";
String task2Name = "task2";
// test submission and event reception
TaskFlowJob job =
(TaskFlowJob)
JobFactory_stax.getFactory()
.createJob(new File(jobDescriptor.toURI()).getAbsolutePath());
if (OperatingSystem.getOperatingSystem() == OperatingSystem.windows) {
((NativeTask) job.getTask(task1Name))
.setCommandLine("cmd", "/C", "ping", "127.0.0.1", "-n", "10", ">", "NUL");
}
JobId id = SchedulerTHelper.submitJob(job);
SchedulerTHelper.log("Job submitted, id " + id.toString());
SchedulerTHelper.log("Waiting for jobSubmitted Event");
JobState receivedState = SchedulerTHelper.waitForEventJobSubmitted(id);
Assert.assertEquals(receivedState.getId(), id);
SchedulerTHelper.log("Waiting for job running");
JobInfo jInfo = SchedulerTHelper.waitForEventJobRunning(id);
Assert.assertEquals(jInfo.getJobId(), id);
Assert.assertEquals(JobStatus.RUNNING, jInfo.getStatus());
SchedulerTHelper.waitForEventTaskRunning(id, task1Name);
TaskInfo tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task1Name);
Assert.assertEquals(TaskStatus.FINISHED, tInfo.getStatus());
SchedulerTHelper.waitForEventTaskRunning(id, task2Name);
tInfo = SchedulerTHelper.waitForEventTaskFinished(id, task2Name);
Assert.assertEquals(TaskStatus.FAULTY, tInfo.getStatus());
SchedulerTHelper.waitForEventJobFinished(id);
JobResult res = SchedulerTHelper.getJobResult(id);
// check that there is one exception in results
Assert.assertTrue(res.getExceptionResults().size() == 1);
// remove job
SchedulerTHelper.removeJob(id);
SchedulerTHelper.waitForEventJobRemoved(id);
}
private String getTo() throws JobEmailNotificationException {
String to = jobState.getGenericInformation().get(GENERIC_INFORMATION_KEY_EMAIL);
if (to == null) {
throw new JobEmailNotificationException(
"Recipient address is not set in generic information");
}
return to;
}
/**
* This method will check, for each awaited job, if the result is available on the Scheduler. If
* positive, the will call the performPostTreatment method in order to perform the post treatment.
*/
protected void checkResultsForAwaitedJobs() {
// we make a copy of the awaitedJobsIds set in order to iterate over it.
Set<AwaitedJob> awaitedJobsIdsCopy = new HashSet<AwaitedJob>(awaitedJobs.values());
Iterator<AwaitedJob> it = awaitedJobsIdsCopy.iterator();
while (it.hasNext()) {
AwaitedJob awaitedJob = it.next();
String id = awaitedJob.getJobId();
try {
JobState js = uischeduler.getJobState(id);
if (js.isFinished()) {
pullData(awaitedJob);
}
} catch (NotConnectedException e) {
logger.error(
"A connection error occured while trying to download output data of Job "
+ id
+ ". This job will remain in the list of awaited jobs. Another attempt to dowload the output data will be made next time the application is initialized. ",
e);
} catch (UnknownJobException e) {
logger.error(
"Could not retrieve output data for job "
+ id
+ " because this job is not known by the Scheduler. \n ",
e);
logger.warn(
"Job "
+ id
+ " will be removed from the known job list. The system will not attempt again to retrieve data for this job. You could try to manually copy the data from the location "
+ awaitedJob.getPullURL());
removeAwaitedJob(id);
} catch (PermissionException e) {
logger.error(
"Could not retrieve output data for job "
+ id
+ " because you don't have permmission to access this job. You need to use the same connection credentials you used for submitting the job. \n Another attempt to dowload the output data for this job will be made next time the application is initialized. ",
e);
}
}
}
private String getBody() {
String status = jobState.getStatus().toString();
String version = Main.version;
String hostname = "UNKNOWN";
try {
hostname = InetAddress.getLocalHost().getCanonicalHostName();
} catch (UnknownHostException e) {
logger.debug("Could not get hostname", e);
}
return String.format(BODY_TEMPLATE, status, version, hostname);
}
@Before
public void setUp() throws Exception {
if (jobId == null) {
scheduler = RestFuncTHelper.getScheduler();
SchedulerState state = scheduler.getState();
List<JobState> jobStates = new ArrayList<>();
jobStates.addAll(state.getPendingJobs());
jobStates.addAll(state.getRunningJobs());
jobStates.addAll(state.getFinishedJobs());
for (JobState jobState : jobStates) {
JobId jobId = jobState.getId();
scheduler.killJob(jobId);
scheduler.removeJob(jobId);
}
// submit a job with a loop and out and err outputs
System.out.println("submit a job with loop, out and err outputs");
jobId = submitJob("flow_loop_out.xml");
}
}
private JobInfo waitForJobEvent(
JobId id, long timeout, JobStatus jobStatusAfterEvent, SchedulerEvent jobEvent)
throws Exception {
JobState jobState = null;
try {
jobState = getSchedulerInterface().getJobState(id);
} catch (UnknownJobException ignored) {
}
if (jobState != null && jobState.getStatus().equals(jobStatusAfterEvent)) {
System.err.println("Job is already finished - do not wait for the 'job finished' event");
return jobState.getJobInfo();
} else {
try {
System.err.println("Waiting for the job finished event");
return getSchedulerMonitorsHandler().waitForEventJob(jobEvent, id, timeout);
} catch (ProActiveTimeoutException e) {
// unreachable block, 0 means infinite, no timeout
// log something ?
return null;
}
}
}
public boolean doCheckAndSend() throws JobEmailNotificationException {
String jobStatus =
jobState.getGenericInformation().get(GENERIC_INFORMATION_KEY_NOTIFICATION_EVENT);
List<String> jobStatusList = new ArrayList<>();
if (jobStatus != null) {
jobStatusList = Arrays.asList(jobStatus.toLowerCase().split("\\s*,\\s*"));
}
switch (eventType) {
case JOB_CHANGE_PRIORITY:
case JOB_IN_ERROR:
case JOB_PAUSED:
case JOB_PENDING_TO_FINISHED:
case JOB_PENDING_TO_RUNNING:
case JOB_RESTARTED_FROM_ERROR:
case JOB_RESUMED:
case JOB_RUNNING_TO_FINISHED:
case JOB_SUBMITTED:
break;
default:
logger.trace("Event not in the list of email notification, doing nothing");
return false;
}
if (!PASchedulerProperties.EMAIL_NOTIFICATIONS_ENABLED.getValueAsBoolean()) {
logger.debug("Notification emails disabled, doing nothing");
return false;
}
if (!jobStatusList.contains(eventType.toString().toLowerCase())) {
return false;
}
try {
sender.sender(getTo(), getSubject(), getBody());
return true;
} catch (AddressException e) {
throw new JobEmailNotificationException("Malformed email address", e);
} catch (MessagingException e) {
throw new JobEmailNotificationException("Error sending email: " + e.getMessage(), e);
}
}
private String getSubject() {
String jobID = jobState.getId().value();
String event = eventType.toString();
return String.format(SUBJECT_TEMPLATE, jobID, event);
}
/**
* Check if the job concerned by this notification is awaited. Retrieve corresponding data if
* needed
*
* @param notification
*/
protected void update(NotificationData<?> notification) {
// am I interested in this job?
JobId id = ((NotificationData<JobInfo>) notification).getData().getJobId();
AwaitedJob aj = awaitedJobs.get(id.toString());
if (aj == null) return;
JobState jobState = null;
try {
jobState = this.getJobState(id);
} catch (NotConnectedException e) {
logger.error("Could not retreive output data for job " + id, e);
} catch (UnknownJobException e) {
logger.error("Could not retreive output data for job " + id, e);
} catch (PermissionException e) {
logger.error(
"Could not retreive output data for job "
+ id
+ ". Did you connect with a diffrent user ? ",
e);
}
if (jobState == null) {
logger.warn(
"The job "
+ id
+ " is listed as awaited but is unknown bby the scheduler. It will be removed from local list");
removeAwaitedJob(id.toString());
}
JobStatus status = jobState.getStatus();
switch (status) {
case KILLED:
{
logger.info("The job " + id + "has been killed. No data will be transfered");
removeAwaitedJob(id.toString());
break;
}
case FINISHED:
{
logger.info("Transfering data for finished job " + id);
pullData(aj);
this.removeAwaitedJob(id.toString());
logger.info("Data transfer finished for finished job " + id);
break;
}
case CANCELED:
{
logger.info("Transfering data for canceled job " + id);
pullData(aj);
this.removeAwaitedJob(id.toString());
break;
}
case FAILED:
{
logger.info("Transfering data for failed job " + id);
pullData(aj);
this.removeAwaitedJob(id.toString());
break;
}
}
}
