@Override
protected void verifyPrecondition() throws CommandException, PreconditionException {
if ((wfAction == null) || (wfAction.isComplete() && wfAction.isPending())) {
if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
&& wfJob.getStatus() != WorkflowJob.Status.PREP) {
throw new PreconditionException(ErrorCode.E0813, wfJob.getStatusStr());
}
} else {
throw new PreconditionException(
ErrorCode.E0814, actionId, wfAction.getStatusStr(), wfAction.isPending());
}
}
public void testActionCheckTransientDuringMRAction() throws Exception {
// When using YARN, skip this test because it relies on shutting down the job tracker, which
// isn't used in YARN
if (createJobConf().get("yarn.resourcemanager.address") != null) {
return;
}
services.destroy();
// Make the max number of retries lower so the test won't take as long
final int maxRetries = 2;
setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries));
services = new Services();
// Disable ActionCheckerService so it doesn't interfere by triggering any extra
// ActionCheckXCommands
setClassesToBeExcluded(
services.getConf(), new String[] {"org.apache.oozie.service.ActionCheckerService"});
services.init();
final JPAService jpaService = Services.get().get(JPAService.class);
WorkflowJobBean job0 =
this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
final String jobId = job0.getId();
WorkflowActionBean action0 =
this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP);
final String actionId = action0.getId();
final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId);
new ActionStartXCommand(actionId, "map-reduce").call();
final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd);
String originalLauncherId = action1.getExternalId();
ActionExecutorContext context =
new ActionXCommand.ActionExecutorContext(job0, action1, false, false);
MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
JobConf conf =
actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
String user = conf.get("user.name");
JobClient jobClient =
Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
final RunningJob launcherJob = jobClient.getJob(JobID.forName(originalLauncherId));
waitFor(
120 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return launcherJob.isComplete();
}
});
assertTrue(launcherJob.isSuccessful());
Map<String, String> actionData =
LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
new ActionCheckXCommand(action1.getId()).call();
WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
String originalMapperId = action2.getExternalChildIDs();
assertFalse(originalLauncherId.equals(originalMapperId));
// At this point, the launcher job has finished and the map-reduce action has started (but not
// finished)
// Now, shutdown the job tracker to pretend it has gone down during the map-reduce job
executeWhileJobTrackerIsShutdown(
new ShutdownJobTrackerExecutable() {
@Override
public void execute() throws Exception {
assertEquals(0, action1.getRetries());
new ActionCheckXCommand(actionId).call();
waitFor(
30 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
return (action1a.getRetries() > 0);
}
});
waitFor(
180 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
return (action1a.getRetries() == 0);
}
});
WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd);
assertEquals(0, action1b.getRetries());
assertEquals("START_MANUAL", action1b.getStatusStr());
WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
assertEquals("SUSPENDED", job1.getStatusStr());
// At this point, the action has gotten a transient error, even after maxRetries tries
// so the workflow has been
// SUSPENDED
}
});
// Now, lets bring the job tracker back up and resume the workflow (which will restart the
// current action)
// It should now continue and finish with SUCCEEDED
new ResumeXCommand(jobId).call();
WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
assertEquals("RUNNING", job2.getStatusStr());
sleep(500);
new ActionCheckXCommand(actionId).call();
WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd);
String launcherId = action3.getExternalId();
assertFalse(originalLauncherId.equals(launcherId));
final RunningJob launcherJob2 = jobClient.getJob(JobID.forName(launcherId));
waitFor(
120 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return launcherJob2.isComplete();
}
});
assertTrue(launcherJob2.isSuccessful());
actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
assertTrue(LauncherMapperHelper.hasIdSwap(actionData));
new ActionCheckXCommand(actionId).call();
WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd);
String mapperId = action4.getExternalChildIDs();
assertFalse(originalMapperId.equals(mapperId));
final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId));
waitFor(
120 * 1000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
return mrJob.isComplete();
}
});
assertTrue(mrJob.isSuccessful());
new ActionCheckXCommand(actionId).call();
WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd);
assertEquals("SUCCEEDED", action5.getExternalStatus());
}
@Override
protected Void execute() throws CommandException {
LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
WorkflowInstance workflowInstance = wfJob.getWorkflowInstance();
workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob);
boolean completed = false;
boolean skipAction = false;
if (wfAction == null) {
if (wfJob.getStatus() == WorkflowJob.Status.PREP) {
try {
completed = workflowInstance.start();
} catch (WorkflowException e) {
throw new CommandException(e);
}
wfJob.setStatus(WorkflowJob.Status.RUNNING);
wfJob.setStartTime(new Date());
wfJob.setWorkflowInstance(workflowInstance);
// 1. Add SLA status event for WF-JOB with status STARTED
// 2. Add SLA registration events for all WF_ACTIONS
SLADbXOperations.writeStausEvent(
wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB);
writeSLARegistrationForAllActions(
workflowInstance.getApp().getDefinition(),
wfJob.getUser(),
wfJob.getGroup(),
wfJob.getConf());
queue(new NotificationXCommand(wfJob));
} else {
throw new CommandException(ErrorCode.E0801, wfJob.getId());
}
} else {
String skipVar =
workflowInstance.getVar(
wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
if (skipVar != null) {
skipAction = skipVar.equals("true");
}
try {
completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue());
} catch (WorkflowException e) {
throw new CommandException(e);
}
wfJob.setWorkflowInstance(workflowInstance);
wfAction.resetPending();
if (!skipAction) {
wfAction.setTransition(workflowInstance.getTransition(wfAction.getName()));
}
try {
jpaService.execute(new WorkflowActionUpdateJPAExecutor(wfAction));
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
}
if (completed) {
try {
for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) {
WorkflowActionBean actionToKill;
actionToKill = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToKillId));
actionToKill.setPending();
actionToKill.setStatus(WorkflowActionBean.Status.KILLED);
jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToKill));
queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType()));
}
for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) {
WorkflowActionBean actionToFail =
jpaService.execute(new WorkflowActionGetJPAExecutor(actionToFailId));
actionToFail.resetPending();
actionToFail.setStatus(WorkflowActionBean.Status.FAILED);
SLADbXOperations.writeStausEvent(
wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION);
jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToFail));
}
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
wfJob.setEndTime(new Date());
wfJob.setWorkflowInstance(workflowInstance);
Status slaStatus = Status.SUCCEEDED;
switch (wfJob.getStatus()) {
case SUCCEEDED:
slaStatus = Status.SUCCEEDED;
break;
case KILLED:
slaStatus = Status.KILLED;
break;
case FAILED:
slaStatus = Status.FAILED;
break;
default: // TODO SUSPENDED
break;
}
SLADbXOperations.writeStausEvent(
wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB);
queue(new NotificationXCommand(wfJob));
if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) {
InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation());
}
} else {
for (WorkflowActionBean newAction :
WorkflowStoreService.getStartedActions(workflowInstance)) {
String skipVar =
workflowInstance.getVar(
newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
boolean skipNewAction = false;
if (skipVar != null) {
skipNewAction = skipVar.equals("true");
}
try {
if (skipNewAction) {
WorkflowActionBean oldAction;
oldAction = jpaService.execute(new WorkflowActionGetJPAExecutor(newAction.getId()));
oldAction.setPending();
jpaService.execute(new WorkflowActionUpdateJPAExecutor(oldAction));
queue(new SignalXCommand(jobId, oldAction.getId()));
} else {
newAction.setPending();
String actionSlaXml =
getActionSLAXml(
newAction.getName(),
workflowInstance.getApp().getDefinition(),
wfJob.getConf());
newAction.setSlaXml(actionSlaXml);
jpaService.execute(new WorkflowActionInsertJPAExecutor(newAction));
LOG.debug(
"SignalXCommand: Name: "
+ newAction.getName()
+ ", Id: "
+ newAction.getId()
+ ", Authcode:"
+ newAction.getCred());
queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
}
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
}
}
try {
jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob));
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
XLog.getLog(getClass())
.debug(
"Updated the workflow status to "
+ wfJob.getId()
+ " status ="
+ wfJob.getStatusStr());
if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
&& wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) {
// update coordinator action
new CoordActionUpdateXCommand(wfJob).call();
new WfEndXCommand(wfJob).call(); // To delete the WF temp dir
}
LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
return null;
}
protected void updateSLASummary() throws XException {
updateSLASummary(wfJob.getId(), wfJob.getStartTime(), wfJob.getEndTime(), wfJob.getStatusStr());
}
