/**
* Constructor to create the Coordinator Submit Command by bundle job.
*
* @param conf : Configuration for Coordinator job
* @param authToken : To be used for authentication
* @param bundleId : bundle id
* @param coordName : coord name
*/
public CoordSubmitXCommand(
Configuration conf, String authToken, String bundleId, String coordName) {
super("coord_submit", "coord_submit", 1);
this.conf = ParamChecker.notNull(conf, "conf");
this.authToken = ParamChecker.notEmpty(authToken, "authToken");
this.bundleId = ParamChecker.notEmpty(bundleId, "bundleId");
this.coordName = ParamChecker.notEmpty(coordName, "coordName");
}
/**
* The constructor for class {@link CoordRerunXCommand}
*
* @param jobId the job id
* @param rerunType rerun type {@link RestConstants.JOB_COORD_RERUN_DATE} or {@link
* RestConstants.JOB_COORD_RERUN_ACTION}
* @param scope the rerun scope for given rerunType separated by ","
* @param refresh true if user wants to refresh input/output dataset urls
* @param noCleanup false if user wants to cleanup output events for given rerun actions
* @param failed true if user wants to rerun only failed nodes
* @param actionRunConf configuration values for actions
*/
public CoordRerunXCommand(
String jobId,
String rerunType,
String scope,
boolean refresh,
boolean noCleanup,
boolean failed,
Configuration actionRunConf) {
super("coord_rerun", "coord_rerun", 1);
this.jobId = ParamChecker.notEmpty(jobId, "jobId");
this.rerunType = ParamChecker.notEmpty(rerunType, "rerunType");
this.scope = ParamChecker.notEmpty(scope, "scope");
this.refresh = refresh;
this.noCleanup = noCleanup;
this.failed = failed;
this.actionRunConf = actionRunConf;
}
/**
* Merge default configuration with user-defined configuration.
*
* @throws CommandException thrown if failed to read or merge configurations
*/
protected void mergeDefaultConfig() throws CommandException {
Path configDefault = null;
try {
String coordAppPathStr = conf.get(OozieClient.COORDINATOR_APP_PATH);
Path coordAppPath = new Path(coordAppPathStr);
String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
String group =
ParamChecker.notEmpty(conf.get(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
FileSystem fs =
Services.get()
.get(HadoopAccessorService.class)
.createFileSystem(user, group, coordAppPath.toUri(), new Configuration());
// app path could be a directory
if (!fs.isFile(coordAppPath)) {
configDefault = new Path(coordAppPath, CONFIG_DEFAULT);
} else {
configDefault = new Path(coordAppPath.getParent(), CONFIG_DEFAULT);
}
if (fs.exists(configDefault)) {
Configuration defaultConf = new XConfiguration(fs.open(configDefault));
PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
XConfiguration.injectDefaults(defaultConf, conf);
} else {
LOG.info("configDefault Doesn't exist " + configDefault);
}
PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
// Resolving all variables in the job properties.
// This ensures the Hadoop Configuration semantics is preserved.
XConfiguration resolvedVarsConf = new XConfiguration();
for (Map.Entry<String, String> entry : conf) {
resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
}
conf = resolvedVarsConf;
} catch (IOException e) {
throw new CommandException(
ErrorCode.E0702,
e.getMessage() + " : Problem reading default config " + configDefault,
e);
} catch (HadoopAccessorException e) {
throw new CommandException(e);
}
LOG.debug("Merged CONF :" + XmlUtils.prettyPrint(conf).toString());
}
/**
* Read the application XML and validate against coordinator Schema
*
* @return validated coordinator XML
* @throws CoordinatorJobException thrown if unable to read or validate coordinator xml
*/
private String readAndValidateXml() throws CoordinatorJobException {
String appPath =
ParamChecker.notEmpty(
conf.get(OozieClient.COORDINATOR_APP_PATH), OozieClient.COORDINATOR_APP_PATH);
String coordXml = readDefinition(appPath);
validateXml(coordXml);
return coordXml;
}
/**
* Return a {@link org.apache.oozie.client.OozieClient} for LocalOozie configured for a given
* user.
*
* <p>The following methods of the client are NOP in the returned instance: {@link
* org.apache.oozie.client.OozieClient#validateWSVersion}, {@link
* org.apache.oozie.client.OozieClient#setHeader}, {@link
* org.apache.oozie.client.OozieClient#getHeader}, {@link
* org.apache.oozie.client.OozieClient#removeHeader}, {@link
* org.apache.oozie.client.OozieClient#getHeaderNames} and {@link
* org.apache.oozie.client.OozieClient#setSafeMode}.
*
* @param user user name to use in LocalOozie for running workflows.
* @return a {@link org.apache.oozie.client.OozieClient} for LocalOozie configured for the given
* user.
*/
public static OozieClient getClient(String user) {
if (!localOozieActive) {
throw new IllegalStateException("LocalOozie is not initialized");
}
ParamChecker.notEmpty(user, "user");
DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, "undef");
return new LocalOozieClient(dagEngine);
}
/**
* Read coordinator definition.
*
* @param appPath application path.
* @return coordinator definition.
* @throws CoordinatorJobException thrown if the definition could not be read.
*/
protected String readDefinition(String appPath) throws CoordinatorJobException {
String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
String group = ParamChecker.notEmpty(conf.get(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
// Configuration confHadoop = CoordUtils.getHadoopConf(conf);
try {
URI uri = new URI(appPath);
LOG.debug("user ="******" group =" + group);
FileSystem fs =
Services.get()
.get(HadoopAccessorService.class)
.createFileSystem(user, group, uri, new Configuration());
Path appDefPath = null;
// app path could be a directory
Path path = new Path(uri.getPath());
// check file exists for dataset include file, app xml already checked
if (!fs.exists(path)) {
throw new URISyntaxException(path.toString(), "path not existed : " + path.toString());
}
if (!fs.isFile(path)) {
appDefPath = new Path(path, COORDINATOR_XML_FILE);
} else {
appDefPath = path;
}
Reader reader = new InputStreamReader(fs.open(appDefPath));
StringWriter writer = new StringWriter();
IOUtils.copyCharStream(reader, writer);
return writer.toString();
} catch (IOException ex) {
LOG.warn("IOException :" + XmlUtils.prettyPrint(conf), ex);
throw new CoordinatorJobException(ErrorCode.E1001, ex.getMessage(), ex);
} catch (URISyntaxException ex) {
LOG.warn("URISyException :" + ex.getMessage());
throw new CoordinatorJobException(ErrorCode.E1002, appPath, ex.getMessage(), ex);
} catch (HadoopAccessorException ex) {
throw new CoordinatorJobException(ex);
} catch (Exception ex) {
LOG.warn("Exception :", ex);
throw new CoordinatorJobException(ErrorCode.E1001, ex.getMessage(), ex);
}
}
/**
* Get the currently missing and available dependencies after checking the list of known missing
* dependencies against the source.
*
* @param missingDependencies known missing dependencies
* @param actionConf Configuration for the action
* @param stopOnFirstMissing Does not continue check for the rest of list if there is a missing
* dependency
* @return ActionDependency which has the list of missing and available dependencies
* @throws CommandException
*/
public static ActionDependency checkForAvailability(
String[] missingDependencies, Configuration actionConf, boolean stopOnFirstMissing)
throws CommandException {
final XLog LOG =
XLog.getLog(DependencyChecker.class); // OOZIE-1251. Don't initialize as static variable.
String user =
ParamChecker.notEmpty(actionConf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
List<String> missingDeps = new ArrayList<String>();
List<String> availableDeps = new ArrayList<String>();
URIHandlerService uriService = Services.get().get(URIHandlerService.class);
boolean continueChecking = true;
try {
for (int index = 0; index < missingDependencies.length; index++) {
if (continueChecking) {
String dependency = missingDependencies[index];
URI uri = new URI(dependency);
URIHandler uriHandler = uriService.getURIHandler(uri);
LOG.debug("Checking for the availability of dependency [{0}] ", dependency);
if (uriHandler.exists(uri, actionConf, user)) {
LOG.debug("Dependency [{0}] is available", dependency);
availableDeps.add(dependency);
} else {
LOG.debug("Dependency [{0}] is missing", dependency);
missingDeps.add(dependency);
if (stopOnFirstMissing) {
continueChecking = false;
}
}
} else {
missingDeps.add(missingDependencies[index]);
}
}
} catch (URISyntaxException e) {
throw new CommandException(ErrorCode.E0906, e.getMessage(), e);
} catch (URIHandlerException e) {
throw new CommandException(e);
}
return new ActionDependency(missingDeps, availableDeps);
}
public BundleJobSuspendXCommand(String id) {
super("bundle_suspend", "bundle_suspend", 1);
this.jobId = ParamChecker.notEmpty(id, "id");
}
public synchronized boolean signal(String executionPath, String signalValue)
throws WorkflowException {
ParamChecker.notEmpty(executionPath, "executionPath");
ParamChecker.notNull(signalValue, "signalValue");
log.debug(
XLog.STD,
"Signaling job execution path [{0}] signal value [{1}]",
executionPath,
signalValue);
if (status != Status.RUNNING) {
throw new WorkflowException(ErrorCode.E0716);
}
NodeInstance nodeJob = executionPaths.get(executionPath);
if (nodeJob == null) {
status = Status.FAILED;
log.error("invalid execution path [{0}]", executionPath);
}
NodeDef nodeDef = null;
if (!status.isEndState()) {
nodeDef = def.getNode(nodeJob.nodeName);
if (nodeDef == null) {
status = Status.FAILED;
log.error("invalid transition [{0}]", nodeJob.nodeName);
}
}
if (!status.isEndState()) {
NodeHandler nodeHandler = newInstance(nodeDef.getHandlerClass());
boolean exiting = true;
Context context = new Context(nodeDef, executionPath, signalValue);
if (!nodeJob.started) {
try {
nodeHandler.loopDetection(context);
exiting = nodeHandler.enter(context);
nodeJob.started = true;
} catch (WorkflowException ex) {
status = Status.FAILED;
List<String> killedNodes = terminateNodes(Status.KILLED);
if (killedNodes.size() > 1) {
log.warn(
XLog.STD,
"Workflow completed [{0}], killing [{1}] running nodes",
status,
killedNodes.size());
}
throw ex;
}
}
if (exiting) {
List<String> pathsToStart = new ArrayList<String>();
List<String> fullTransitions;
try {
fullTransitions = nodeHandler.multiExit(context);
int last = fullTransitions.size() - 1;
// TEST THIS
if (last >= 0) {
String transitionTo = getTransitionNode(fullTransitions.get(last));
if (nodeDef instanceof ForkNodeDef) {
transitionTo = "*"; // WF action cannot hold all transitions for a fork.
// transitions are hardcoded in the WF app.
}
persistentVars.put(
nodeDef.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + TRANSITION_TO,
transitionTo);
}
} catch (WorkflowException ex) {
status = Status.FAILED;
throw ex;
}
if (context.status == Status.KILLED) {
status = Status.KILLED;
log.debug(XLog.STD, "Completing job, kill node [{0}]", nodeJob.nodeName);
} else {
if (context.status == Status.FAILED) {
status = Status.FAILED;
log.debug(XLog.STD, "Completing job, fail node [{0}]", nodeJob.nodeName);
} else {
if (context.status == Status.SUCCEEDED) {
status = Status.SUCCEEDED;
log.debug(XLog.STD, "Completing job, end node [{0}]", nodeJob.nodeName);
}
/*
else if (context.status == Status.SUSPENDED) {
status = Status.SUSPENDED;
log.debug(XLog.STD, "Completing job, end node [{0}]", nodeJob.nodeName);
}
*/
else {
for (String fullTransition : fullTransitions) {
// this is the whole trick for forking, we need the
// executionpath and the transition
// in the case of no forking last element of
// executionpath is different from transition
// in the case of forking they are the same
log.debug(
XLog.STD,
"Exiting node [{0}] with transition[{1}]",
nodeJob.nodeName,
fullTransition);
String execPathFromTransition = getExecutionPath(fullTransition);
String transition = getTransitionNode(fullTransition);
def.validateTransition(nodeJob.nodeName, transition);
NodeInstance nodeJobInPath = executionPaths.get(execPathFromTransition);
if ((nodeJobInPath == null) || (!transition.equals(nodeJobInPath.nodeName))) {
// TODO explain this IF better
// If the WfJob is signaled with the parent
// execution executionPath again
// The Fork node will execute again.. and replace
// the Node WorkflowJobBean
// so this is required to prevent that..
// Question : Should we throw an error in this case
// ??
executionPaths.put(execPathFromTransition, new NodeInstance(transition));
pathsToStart.add(execPathFromTransition);
}
}
// signal all new synch transitions
for (String pathToStart : pathsToStart) {
signal(pathToStart, "::synch::");
}
}
}
}
}
}
if (status.isEndState()) {
if (status == Status.FAILED) {
List<String> failedNodes = terminateNodes(status);
log.warn(
XLog.STD,
"Workflow completed [{0}], failing [{1}] running nodes",
status,
failedNodes.size());
} else {
List<String> killedNodes = terminateNodes(Status.KILLED);
if (killedNodes.size() > 1) {
log.warn(
XLog.STD,
"Workflow completed [{0}], killing [{1}] running nodes",
status,
killedNodes.size());
}
}
}
return status.isEndState();
}
/**
* @param id bundle job id
* @param changeValue change value
* @throws CommandException thrown if failed to change bundle
*/
public BundleJobChangeXCommand(String id, String changeValue) throws CommandException {
super("bundle_change", "bundle_change", 1);
this.jobId = ParamChecker.notEmpty(id, "id");
this.changeValue = ParamChecker.notEmpty(changeValue, "changeValue");
}
public SignalXCommand(String jobId, String actionId) {
this("signal", 1, jobId);
this.actionId = ParamChecker.notEmpty(actionId, "actionId");
}
public SignalXCommand(String name, int priority, String jobId) {
super(name, name, priority);
this.jobId = ParamChecker.notEmpty(jobId, "jobId");
}
/**
* @param offset
* @return n-th available latest instance Date-Time for SYNC data-set
* @throws Exception
*/
private static String coord_latest_sync(int offset) throws Exception {
if (offset > 0) {
throw new RuntimeException(
"For latest there is no meaning " + "of positive instance. n should be <=0" + offset);
}
ELEvaluator eval = ELEvaluator.getCurrent();
String retVal = "";
int datasetFrequency = (int) getDSFrequency(); // in minutes
TimeUnit dsTimeUnit = getDSTimeUnit();
int[] instCount = new int[1];
Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
if (nominalInstanceCal != null) {
Calendar initInstance = getInitialInstanceCal();
SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
if (ds == null) {
throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
}
String uriTemplate = ds.getUriTemplate();
Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
if (conf == null) {
throw new RuntimeException(
"Associated Configuration should be defined with key " + CONFIGURATION);
}
int available = 0;
boolean resolved = false;
String user =
ParamChecker.notEmpty(
(String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
String group =
ParamChecker.notEmpty(
(String) eval.getVariable(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
String doneFlag = ds.getDoneFlag();
while (nominalInstanceCal.compareTo(initInstance) >= 0) {
ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
String uriPath = uriEval.evaluate(uriTemplate, String.class);
String pathWithDoneFlag = uriPath;
if (doneFlag.length() > 0) {
pathWithDoneFlag += "/" + doneFlag;
}
if (isPathAvailable(pathWithDoneFlag, user, group, conf)) {
XLog.getLog(CoordELFunctions.class)
.debug("Found latest(" + available + "): " + pathWithDoneFlag);
if (available == offset) {
XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag);
resolved = true;
retVal = DateUtils.formatDateUTC(nominalInstanceCal);
eval.setVariable("resolved_path", uriPath);
break;
}
available--;
}
// nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(),
// -datasetFrequency);
nominalInstanceCal = (Calendar) initInstance.clone();
instCount[0]--;
nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
// DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
}
if (!resolved) {
// return unchanged latest function with variable 'is_resolved'
// to 'false'
eval.setVariable("is_resolved", Boolean.FALSE);
retVal = "${coord:latest(" + offset + ")}";
} else {
eval.setVariable("is_resolved", Boolean.TRUE);
}
} else { // No feasible nominal time
eval.setVariable("is_resolved", Boolean.FALSE);
}
return retVal;
}