/*
* protected method to get a job in JsonBean representation
*/
@Override
protected JsonBean getJob(HttpServletRequest request, HttpServletResponse response)
throws XServletException, IOException, BaseEngineException {
ServletInputStream is = request.getInputStream();
byte[] b = new byte[101];
while (is.readLine(b, 0, 100) != -1) {
XLog.getLog(getClass()).warn("Printing :" + new String(b));
}
JsonBean jobBean = null;
String jobId = getResourceName(request);
if (jobId.endsWith("-B")) {
jobBean = getBundleJob(request, response);
} else {
if (jobId.endsWith("-W")) {
jobBean = getWorkflowJob(request, response);
} else {
if (jobId.contains("-W@")) {
jobBean = getWorkflowAction(request, response);
} else {
if (jobId.contains("-C@")) {
jobBean = getCoordinatorAction(request, response);
} else {
jobBean = getCoordinatorJob(request, response);
}
}
}
}
return jobBean;
}
private HCatClient getHCatClient(URI uri, Configuration conf, String user)
throws HCatAccessorException {
final HiveConf hiveConf = new HiveConf(conf, this.getClass());
String serverURI = getMetastoreConnectURI(uri);
if (!serverURI.equals("")) {
hiveConf.set("hive.metastore.local", "false");
}
hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, serverURI);
try {
XLog.getLog(HCatURIHandler.class)
.info(
"Creating HCatClient for user [{0}] login_user [{1}] and server [{2}] ",
user, UserGroupInformation.getLoginUser(), serverURI);
// HiveMetastoreClient (hive 0.9) currently does not work if UGI has doAs
// We are good to connect as the oozie user since listPartitions does not require
// authorization
/*
UserGroupInformation ugi = ugiService.getProxyUser(user);
return ugi.doAs(new PrivilegedExceptionAction<HCatClient>() {
public HCatClient run() throws Exception {
return HCatClient.create(hiveConf);
}
});
*/
return HCatClient.create(hiveConf);
} catch (HCatException e) {
throw new HCatAccessorException(ErrorCode.E1501, e);
} catch (IOException e) {
throw new HCatAccessorException(ErrorCode.E1501, e);
}
}
/**
* Rerun bundle job
*
* @param request servlet request
* @param response servlet response
* @param conf configration object
* @throws XServletException
*/
private void rerunBundleJob(
HttpServletRequest request, HttpServletResponse response, Configuration conf)
throws XServletException {
JSONObject json = new JSONObject();
BundleEngine bundleEngine =
Services.get()
.get(BundleEngineService.class)
.getBundleEngine(getUser(request), getAuthToken(request));
String jobId = getResourceName(request);
String coordScope = request.getParameter(RestConstants.JOB_BUNDLE_RERUN_COORD_SCOPE_PARAM);
String dateScope = request.getParameter(RestConstants.JOB_BUNDLE_RERUN_DATE_SCOPE_PARAM);
String refresh = request.getParameter(RestConstants.JOB_COORD_RERUN_REFRESH_PARAM);
String noCleanup = request.getParameter(RestConstants.JOB_COORD_RERUN_NOCLEANUP_PARAM);
XLog.getLog(getClass())
.info(
"Rerun Bundle for jobId="
+ jobId
+ ", coordScope="
+ coordScope
+ ", dateScope="
+ dateScope
+ ", refresh="
+ refresh
+ ", noCleanup="
+ noCleanup);
try {
bundleEngine.reRun(
jobId, coordScope, dateScope, Boolean.valueOf(refresh), Boolean.valueOf(noCleanup));
} catch (BaseEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
}
/**
* Set and set system mode.
*
* @param .
*/
public synchronized void setSystemMode(SYSTEM_MODE sysMode) {
if (this.systemMode != sysMode) {
XLog log = XLog.getLog(getClass());
log.info(XLog.OPS, "Exiting " + this.systemMode + " Entering " + sysMode);
}
this.systemMode = sysMode;
}
/**
* Initialize all services define in the {@link #CONF_SERVICE_CLASSES} configuration property.
*
* @throws ServiceException thrown if any of the services could not initialize.
*/
@SuppressWarnings("unchecked")
public void init() throws ServiceException {
XLog log = new XLog(LogFactory.getLog(getClass()));
log.trace("Initializing");
SERVICES = this;
try {
Class<? extends Service>[] serviceClasses =
(Class<? extends Service>[]) conf.getClasses(CONF_SERVICE_CLASSES);
if (serviceClasses != null) {
for (Class<? extends Service> serviceClass : serviceClasses) {
setService(serviceClass);
}
}
} catch (RuntimeException ex) {
XLog.getLog(getClass()).fatal(ex.getMessage(), ex);
throw ex;
} catch (ServiceException ex) {
SERVICES = null;
throw ex;
}
InstrumentationService instrService = get(InstrumentationService.class);
if (instrService != null) {
for (Service service : services.values()) {
if (service instanceof Instrumentable) {
((Instrumentable) service).instrument(instrService.get());
}
}
}
log.info("Initialized");
log.info("Oozie System ID [{0}] started!", getSystemId());
}
/**
* Start LocalOozie.
*
* @throws Exception if LocalOozie could not be started.
*/
public static synchronized void start() throws Exception {
if (localOozieActive) {
throw new IllegalStateException("LocalOozie is already initialized");
}
String log4jFile = System.getProperty(XLogService.LOG4J_FILE_ENV, null);
String oozieLocalLog = System.getProperty("oozielocal.log", null);
if (log4jFile == null) {
System.setProperty(XLogService.LOG4J_FILE_ENV, "localoozie-log4j.properties");
}
if (oozieLocalLog == null) {
System.setProperty("oozielocal.log", "./oozielocal.log");
}
localOozieActive = true;
new Services().init();
if (log4jFile != null) {
System.setProperty(XLogService.LOG4J_FILE_ENV, log4jFile);
} else {
System.getProperties().remove(XLogService.LOG4J_FILE_ENV);
}
if (oozieLocalLog != null) {
System.setProperty("oozielocal.log", oozieLocalLog);
} else {
System.getProperties().remove("oozielocal.log");
}
container = new EmbeddedServletContainer("oozie");
container.addServletEndpoint("/callback", CallbackServlet.class);
container.start();
String callbackUrl = container.getServletURL("/callback");
Services.get().getConf().set(CallbackService.CONF_BASE_URL, callbackUrl);
XLog.getLog(LocalOozie.class).info("LocalOozie started callback set to [{0}]", callbackUrl);
}
@Override
public void destroy() {
try {
hcatClient.close();
} catch (Exception ignore) {
XLog.getLog(HCatContext.class).warn("Error closing hcat client", ignore);
}
}
private String createRuntimeDir() throws ServiceException {
try {
File file = File.createTempFile(getSystemId(), ".dir");
file.delete();
if (!file.mkdir()) {
ServiceException ex = new ServiceException(ErrorCode.E0001, file.getAbsolutePath());
XLog.getLog(getClass()).fatal(ex);
throw ex;
}
XLog.getLog(getClass()).info("Initialized runtime directory [{0}]", file.getAbsolutePath());
return file.getAbsolutePath();
} catch (IOException ex) {
ServiceException sex = new ServiceException(ErrorCode.E0001, ex);
XLog.getLog(getClass()).fatal(ex);
throw sex;
}
}
private void refreshLog() {
XLog.Info.get().setParameter(XLogService.USER, conf.get(OozieClient.USER_NAME));
XLog.Info.get().setParameter(XLogService.GROUP, conf.get(OozieClient.GROUP_NAME));
XLog.Info.get().setParameter(DagXLogInfoService.APP, def.getName());
XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN, ""));
XLog.Info.get().setParameter(DagXLogInfoService.JOB, instanceId);
log = XLog.getLog(getClass());
}
private void closeQuietly(HCatClient client, boolean close) {
if (close && client != null) {
try {
client.close();
} catch (Exception ignore) {
XLog.getLog(HCatURIHandler.class).warn("Error closing hcat client", ignore);
}
}
}
public static String ph1_coord_dataOut_echo(String n) {
ELEvaluator eval = ELEvaluator.getCurrent();
String val = (String) eval.getVariable("oozie.dataname." + n);
if (val == null || val.equals("data-out") == false) {
XLog.getLog(CoordELFunctions.class).error("data_out_name " + n + " is not valid");
throw new RuntimeException("data_out_name " + n + " is not valid");
}
return echoUnResolved("dataOut", "'" + n + "'");
}
@Override
protected Void execute(WorkflowStore store) throws CommandException, StoreException {
XLog.getLog(getClass()).debug("STARTED ActionEndCommand for action " + id);
try {
jobId = Services.get().get(UUIDService.class).getId(id);
if (lock(jobId)) {
call(store);
} else {
queueCallable(new ActionEndCommand(id, type), LOCK_FAILURE_REQUEUE_INTERVAL);
XLog.getLog(getClass()).warn("ActionEnd lock was not acquired - failed {0}", id);
}
} catch (InterruptedException e) {
queueCallable(new ActionEndCommand(id, type), LOCK_FAILURE_REQUEUE_INTERVAL);
XLog.getLog(getClass())
.warn("ActionEnd lock was not acquired - interrupted exception failed {0}", id);
} finally {
XLog.getLog(getClass()).debug("ENDED ActionEndCommand for action " + id);
}
return null;
}
/**
* Rerun coordinator actions
*
* @param request servlet request
* @param response servlet response
* @param conf configuration object
* @throws XServletException
*/
@SuppressWarnings("unchecked")
private JSONObject reRunCoordinatorActions(
HttpServletRequest request, HttpServletResponse response, Configuration conf)
throws XServletException {
JSONObject json = new JSONObject();
CoordinatorEngine coordEngine =
Services.get()
.get(CoordinatorEngineService.class)
.getCoordinatorEngine(getUser(request), getAuthToken(request));
String jobId = getResourceName(request);
String rerunType = request.getParameter(RestConstants.JOB_COORD_RERUN_TYPE_PARAM);
String scope = request.getParameter(RestConstants.JOB_COORD_RERUN_SCOPE_PARAM);
String refresh = request.getParameter(RestConstants.JOB_COORD_RERUN_REFRESH_PARAM);
String noCleanup = request.getParameter(RestConstants.JOB_COORD_RERUN_NOCLEANUP_PARAM);
XLog.getLog(getClass())
.info(
"Rerun coordinator for jobId="
+ jobId
+ ", rerunType="
+ rerunType
+ ",scope="
+ scope
+ ",refresh="
+ refresh
+ ", noCleanup="
+ noCleanup);
try {
if (!(rerunType.equals(RestConstants.JOB_COORD_RERUN_DATE)
|| rerunType.equals(RestConstants.JOB_COORD_RERUN_ACTION))) {
throw new CommandException(ErrorCode.E1018, "date or action expected.");
}
CoordinatorActionInfo coordInfo =
coordEngine.reRun(
jobId, rerunType, scope, Boolean.valueOf(refresh), Boolean.valueOf(noCleanup));
List<CoordinatorActionBean> coordActions;
if (coordInfo != null) {
coordActions = coordInfo.getCoordActions();
} else {
coordActions = CoordRerunXCommand.getCoordActions(rerunType, jobId, scope);
}
json.put(
JsonTags.COORDINATOR_ACTIONS, CoordinatorActionBean.toJSONArray(coordActions, "GMT"));
} catch (BaseEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
} catch (CommandException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
return json;
}
/**
* Create a services.
*
* <p>The built in services are initialized.
*
* @throws ServiceException thrown if any of the built in services could not initialize.
*/
public Services() throws ServiceException {
if (SERVICES != null) {
XLog log = XLog.getLog(getClass());
log.warn(XLog.OPS, "Previous services singleton active, destroying it");
SERVICES.destroy();
SERVICES = null;
}
setServiceInternal(XLogService.class, false);
setServiceInternal(ConfigurationService.class, true);
conf = get(ConfigurationService.class).getConf();
systemId = conf.get(CONF_SYSTEM_ID, ("oozie-" + System.getProperty("user.name")));
if (systemId.length() > MAX_SYSTEM_ID_LEN) {
systemId = systemId.substring(0, MAX_SYSTEM_ID_LEN);
XLog.getLog(getClass())
.warn(
"System ID [{0}] exceeds maximun lenght [{1}], trimming",
systemId, MAX_SYSTEM_ID_LEN);
}
setSystemMode(SYSTEM_MODE.valueOf(conf.get(CONF_SYSTEM_MODE, SYSTEM_MODE.NORMAL.toString())));
runtimeDir = createRuntimeDir();
}
/**
* Get the currently missing and available dependencies after checking the list of known missing
* dependencies against the source.
*
* @param missingDependencies known missing dependencies
* @param actionConf Configuration for the action
* @param stopOnFirstMissing Does not continue check for the rest of list if there is a missing
* dependency
* @return ActionDependency which has the list of missing and available dependencies
* @throws CommandException
*/
public static ActionDependency checkForAvailability(
String[] missingDependencies, Configuration actionConf, boolean stopOnFirstMissing)
throws CommandException {
final XLog LOG =
XLog.getLog(DependencyChecker.class); // OOZIE-1251. Don't initialize as static variable.
String user =
ParamChecker.notEmpty(actionConf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
List<String> missingDeps = new ArrayList<String>();
List<String> availableDeps = new ArrayList<String>();
URIHandlerService uriService = Services.get().get(URIHandlerService.class);
boolean continueChecking = true;
try {
for (int index = 0; index < missingDependencies.length; index++) {
if (continueChecking) {
String dependency = missingDependencies[index];
URI uri = new URI(dependency);
URIHandler uriHandler = uriService.getURIHandler(uri);
LOG.debug("Checking for the availability of dependency [{0}] ", dependency);
if (uriHandler.exists(uri, actionConf, user)) {
LOG.debug("Dependency [{0}] is available", dependency);
availableDeps.add(dependency);
} else {
LOG.debug("Dependency [{0}] is missing", dependency);
missingDeps.add(dependency);
if (stopOnFirstMissing) {
continueChecking = false;
}
}
} else {
missingDeps.add(missingDependencies[index]);
}
}
} catch (URISyntaxException e) {
throw new CommandException(ErrorCode.E0906, e.getMessage(), e);
} catch (URIHandlerException e) {
throw new CommandException(e);
}
return new ActionDependency(missingDeps, availableDeps);
}
/** v1 service implementation to submit a coordinator job */
@SuppressWarnings("unchecked")
private JSONObject submitCoordinatorJob(HttpServletRequest request, Configuration conf)
throws XServletException {
JSONObject json = new JSONObject();
XLog.getLog(getClass()).warn("submitCoordinatorJob " + XmlUtils.prettyPrint(conf).toString());
try {
String action = request.getParameter(RestConstants.ACTION_PARAM);
if (action != null
&& !action.equals(RestConstants.JOB_ACTION_START)
&& !action.equals(RestConstants.JOB_ACTION_DRYRUN)) {
throw new XServletException(
HttpServletResponse.SC_BAD_REQUEST,
ErrorCode.E0303,
RestConstants.ACTION_PARAM,
action);
}
boolean startJob = (action != null);
String user = conf.get(OozieClient.USER_NAME);
CoordinatorEngine coordEngine =
Services.get()
.get(CoordinatorEngineService.class)
.getCoordinatorEngine(user, getAuthToken(request));
String id = null;
boolean dryrun = false;
if (action != null) {
dryrun = (action.equals(RestConstants.JOB_ACTION_DRYRUN));
}
if (dryrun) {
id = coordEngine.dryrunSubmit(conf, startJob);
} else {
id = coordEngine.submitJob(conf, startJob);
}
json.put(JsonTags.JOB_ID, id);
} catch (CoordinatorEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
return json;
}
private void setServiceInternal(Class<? extends Service> klass, boolean logging)
throws ServiceException {
try {
Service newService = (Service) ReflectionUtils.newInstance(klass, null);
Service oldService = services.get(newService.getInterface());
if (oldService != null) {
oldService.destroy();
}
if (logging) {
XLog log = new XLog(LogFactory.getLog(getClass()));
log.trace(
"Initializing service[{0}] class[{1}]",
newService.getInterface(), newService.getClass());
}
newService.init(this);
services.put(newService.getInterface(), newService);
} catch (ServiceException ex) {
XLog.getLog(getClass()).fatal(ex.getMessage(), ex);
destroy();
throw ex;
}
}
/** Stop LocalOozie. */
public static synchronized void stop() {
RuntimeException thrown = null;
try {
if (container != null) {
container.stop();
}
} catch (RuntimeException ex) {
thrown = ex;
}
container = null;
XLog.getLog(LocalOozie.class).info("LocalOozie stopped");
try {
Services.get().destroy();
} catch (RuntimeException ex) {
if (thrown != null) {
thrown = ex;
}
}
localOozieActive = false;
if (thrown != null) {
throw thrown;
}
}
// TODO javadoc
public class LiteWorkflowInstance implements Writable, WorkflowInstance {
private static final String TRANSITION_TO = "transition.to";
private XLog log = XLog.getLog(getClass());
private static String PATH_SEPARATOR = "/";
private static String ROOT = PATH_SEPARATOR;
private static String TRANSITION_SEPARATOR = "#";
// Using unique string to indicate version. This is to make sure that it
// doesn't match with user data.
private static final String DATA_VERSION = "V==1";
private static class NodeInstance {
String nodeName;
boolean started = false;
private NodeInstance(String nodeName) {
this.nodeName = nodeName;
}
}
private class Context implements NodeHandler.Context {
private NodeDef nodeDef;
private String executionPath;
private String exitState;
private Status status = Status.RUNNING;
private Context(NodeDef nodeDef, String executionPath, String exitState) {
this.nodeDef = nodeDef;
this.executionPath = executionPath;
this.exitState = exitState;
}
public NodeDef getNodeDef() {
return nodeDef;
}
public String getExecutionPath() {
return executionPath;
}
public String getParentExecutionPath(String executionPath) {
return LiteWorkflowInstance.getParentPath(executionPath);
}
public String getSignalValue() {
return exitState;
}
public String createExecutionPath(String name) {
return LiteWorkflowInstance.createChildPath(executionPath, name);
}
public String createFullTransition(String executionPath, String transition) {
return LiteWorkflowInstance.createFullTransition(executionPath, transition);
}
public void deleteExecutionPath() {
if (!executionPaths.containsKey(executionPath)) {
throw new IllegalStateException();
}
executionPaths.remove(executionPath);
executionPath = LiteWorkflowInstance.getParentPath(executionPath);
}
public void failJob() {
status = Status.FAILED;
}
public void killJob() {
status = Status.KILLED;
}
public void completeJob() {
status = Status.SUCCEEDED;
}
@Override
public Object getTransientVar(String name) {
return LiteWorkflowInstance.this.getTransientVar(name);
}
@Override
public String getVar(String name) {
return LiteWorkflowInstance.this.getVar(name);
}
@Override
public void setTransientVar(String name, Object value) {
LiteWorkflowInstance.this.setTransientVar(name, value);
}
@Override
public void setVar(String name, String value) {
LiteWorkflowInstance.this.setVar(name, value);
}
@Override
public LiteWorkflowInstance getProcessInstance() {
return LiteWorkflowInstance.this;
}
}
private LiteWorkflowApp def;
private Configuration conf;
private String instanceId;
private Status status;
private Map<String, NodeInstance> executionPaths = new HashMap<String, NodeInstance>();
private Map<String, String> persistentVars = new HashMap<String, String>();
private Map<String, Object> transientVars = new HashMap<String, Object>();
protected LiteWorkflowInstance() {
log = XLog.getLog(getClass());
}
public LiteWorkflowInstance(LiteWorkflowApp def, Configuration conf, String instanceId) {
this();
this.def = ParamChecker.notNull(def, "def");
this.instanceId = ParamChecker.notNull(instanceId, "instanceId");
this.conf = ParamChecker.notNull(conf, "conf");
refreshLog();
status = Status.PREP;
}
public synchronized boolean start() throws WorkflowException {
if (status != Status.PREP) {
throw new WorkflowException(ErrorCode.E0719);
}
log.debug(XLog.STD, "Starting job");
status = Status.RUNNING;
executionPaths.put(ROOT, new NodeInstance(StartNodeDef.START));
return signal(ROOT, StartNodeDef.START);
}
// todo if suspended store signal and use when resuming
public synchronized boolean signal(String executionPath, String signalValue)
throws WorkflowException {
ParamChecker.notEmpty(executionPath, "executionPath");
ParamChecker.notNull(signalValue, "signalValue");
if (status != Status.RUNNING) {
throw new WorkflowException(ErrorCode.E0716);
}
NodeInstance nodeJob = executionPaths.get(executionPath);
log.debug(
XLog.STD,
"Signaling job execution path [{0}] signal value [{1}] for node [{2}]",
executionPath,
signalValue,
(nodeJob == null ? null : nodeJob.nodeName));
if (nodeJob == null) {
status = Status.FAILED;
log.error("invalid execution path [{0}]", executionPath);
}
NodeDef nodeDef = null;
if (!status.isEndState()) {
nodeDef = def.getNode(nodeJob.nodeName);
if (nodeDef == null) {
status = Status.FAILED;
log.error("invalid transition [{0}]", nodeJob.nodeName);
}
}
if (!status.isEndState()) {
NodeHandler nodeHandler = newInstance(nodeDef.getHandlerClass());
boolean exiting = true;
Context context = new Context(nodeDef, executionPath, signalValue);
if (!nodeJob.started) {
try {
nodeHandler.loopDetection(context);
exiting = nodeHandler.enter(context);
nodeJob.started = true;
} catch (WorkflowException ex) {
status = Status.FAILED;
List<String> killedNodes = terminateNodes(Status.KILLED);
if (killedNodes.size() > 1) {
log.warn(
XLog.STD,
"Workflow completed [{0}], killing [{1}] running nodes",
status,
killedNodes.size());
}
throw ex;
}
}
if (exiting) {
List<String> pathsToStart = new ArrayList<String>();
List<String> fullTransitions;
try {
fullTransitions = nodeHandler.multiExit(context);
int last = fullTransitions.size() - 1;
// TEST THIS
if (last >= 0) {
String transitionTo = getTransitionNode(fullTransitions.get(last));
if (nodeDef instanceof ForkNodeDef) {
transitionTo = "*"; // WF action cannot hold all transitions for a fork.
// transitions are hardcoded in the WF app.
}
persistentVars.put(
nodeDef.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + TRANSITION_TO,
transitionTo);
}
} catch (WorkflowException ex) {
status = Status.FAILED;
throw ex;
}
if (context.status == Status.KILLED) {
status = Status.KILLED;
log.debug(XLog.STD, "Completing job, kill node [{0}]", nodeJob.nodeName);
} else if (context.status == Status.FAILED) {
status = Status.FAILED;
log.debug(XLog.STD, "Completing job, fail node [{0}]", nodeJob.nodeName);
} else if (context.status == Status.SUCCEEDED) {
status = Status.SUCCEEDED;
log.debug(XLog.STD, "Completing job, end node [{0}]", nodeJob.nodeName);
} else {
for (String fullTransition : fullTransitions) {
// this is the whole trick for forking, we need the executionpath and the transition.
// in case of no forking, last element of executionpath is different from transition.
// in case of forking, they are the same
log.debug(
XLog.STD,
"Exiting node [{0}] with transition[{1}]",
nodeJob.nodeName,
fullTransition);
String execPathFromTransition = getExecutionPath(fullTransition);
String transition = getTransitionNode(fullTransition);
def.validateTransition(nodeJob.nodeName, transition);
NodeInstance nodeJobInPath = executionPaths.get(execPathFromTransition);
if ((nodeJobInPath == null) || (!transition.equals(nodeJobInPath.nodeName))) {
// TODO explain this IF better
// If the WfJob is signaled with the parent
// execution executionPath again
// The Fork node will execute again.. and replace
// the Node WorkflowJobBean
// so this is required to prevent that..
// Question : Should we throw an error in this case
// ??
executionPaths.put(execPathFromTransition, new NodeInstance(transition));
pathsToStart.add(execPathFromTransition);
}
}
// signal all new synch transitions
for (String pathToStart : pathsToStart) {
signal(pathToStart, "::synch::");
}
}
}
}
if (status.isEndState()) {
if (status == Status.FAILED) {
List<String> failedNodes = terminateNodes(status);
log.warn(
XLog.STD,
"Workflow completed [{0}], failing [{1}] running nodes",
status,
failedNodes.size());
} else {
List<String> killedNodes = terminateNodes(Status.KILLED);
if (killedNodes.size() > 1) {
log.warn(
XLog.STD,
"Workflow completed [{0}], killing [{1}] running nodes",
status,
killedNodes.size());
}
}
}
return status.isEndState();
}
/**
* Get NodeDef from workflow instance
*
* @param executionPath execution path
* @return node def
*/
public NodeDef getNodeDef(String executionPath) {
NodeInstance nodeJob = executionPaths.get(executionPath);
NodeDef nodeDef = null;
if (nodeJob == null) {
log.error("invalid execution path [{0}]", executionPath);
} else {
nodeDef = def.getNode(nodeJob.nodeName);
if (nodeDef == null) {
log.error("invalid transition [{0}]", nodeJob.nodeName);
}
}
return nodeDef;
}
public synchronized void fail(String nodeName) throws WorkflowException {
if (status.isEndState()) {
throw new WorkflowException(ErrorCode.E0718);
}
String failedNode = failNode(nodeName);
if (failedNode != null) {
log.warn(XLog.STD, "Workflow Failed. Failing node [{0}]", failedNode);
} else {
// TODO failed attempting to fail the action. EXCEPTION
}
List<String> killedNodes = killNodes();
if (killedNodes.size() > 1) {
log.warn(XLog.STD, "Workflow Failed, killing [{0}] nodes", killedNodes.size());
}
status = Status.FAILED;
}
public synchronized void kill() throws WorkflowException {
if (status.isEndState()) {
throw new WorkflowException(ErrorCode.E0718);
}
log.debug(XLog.STD, "Killing job");
List<String> killedNodes = killNodes();
if (killedNodes.size() > 1) {
log.warn(XLog.STD, "workflow killed, killing [{0}] nodes", killedNodes.size());
}
status = Status.KILLED;
}
public synchronized void suspend() throws WorkflowException {
if (status != Status.RUNNING) {
throw new WorkflowException(ErrorCode.E0716);
}
log.debug(XLog.STD, "Suspending job");
this.status = Status.SUSPENDED;
}
public boolean isSuspended() {
return (status == Status.SUSPENDED);
}
public synchronized void resume() throws WorkflowException {
if (status != Status.SUSPENDED) {
throw new WorkflowException(ErrorCode.E0717);
}
log.debug(XLog.STD, "Resuming job");
status = Status.RUNNING;
}
public void setVar(String name, String value) {
if (value != null) {
persistentVars.put(name, value);
} else {
persistentVars.remove(name);
}
}
@Override
public Map<String, String> getAllVars() {
return persistentVars;
}
@Override
public void setAllVars(Map<String, String> varMap) {
persistentVars.putAll(varMap);
}
public String getVar(String name) {
return persistentVars.get(name);
}
public void setTransientVar(String name, Object value) {
if (value != null) {
transientVars.put(name, value);
} else {
transientVars.remove(name);
}
}
public boolean hasTransientVar(String name) {
return transientVars.containsKey(name);
}
public Object getTransientVar(String name) {
return transientVars.get(name);
}
public boolean hasEnded() {
return status.isEndState();
}
private List<String> terminateNodes(Status endStatus) {
List<String> endNodes = new ArrayList<String>();
for (Map.Entry<String, NodeInstance> entry : executionPaths.entrySet()) {
if (entry.getValue().started) {
NodeDef nodeDef = def.getNode(entry.getValue().nodeName);
if (!(nodeDef instanceof ControlNodeDef)) {
NodeHandler nodeHandler = newInstance(nodeDef.getHandlerClass());
try {
if (endStatus == Status.KILLED) {
nodeHandler.kill(new Context(nodeDef, entry.getKey(), null));
} else {
if (endStatus == Status.FAILED) {
nodeHandler.fail(new Context(nodeDef, entry.getKey(), null));
}
}
endNodes.add(nodeDef.getName());
} catch (Exception ex) {
log.warn(
XLog.STD,
"Error Changing node state to [{0}] for Node [{1}]",
endStatus.toString(),
nodeDef.getName(),
ex);
}
}
}
}
return endNodes;
}
private String failNode(String nodeName) {
String failedNode = null;
for (Map.Entry<String, NodeInstance> entry : executionPaths.entrySet()) {
String node = entry.getKey();
NodeInstance nodeInstance = entry.getValue();
if (nodeInstance.started && nodeInstance.nodeName.equals(nodeName)) {
NodeDef nodeDef = def.getNode(nodeInstance.nodeName);
NodeHandler nodeHandler = newInstance(nodeDef.getHandlerClass());
try {
nodeHandler.fail(new Context(nodeDef, node, null));
failedNode = nodeDef.getName();
nodeInstance.started = false;
} catch (Exception ex) {
log.warn(XLog.STD, "Error failing node [{0}]", nodeDef.getName(), ex);
}
return failedNode;
}
}
return failedNode;
}
private List<String> killNodes() {
List<String> killedNodes = new ArrayList<String>();
for (Map.Entry<String, NodeInstance> entry : executionPaths.entrySet()) {
String node = entry.getKey();
NodeInstance nodeInstance = entry.getValue();
if (nodeInstance.started) {
NodeDef nodeDef = def.getNode(nodeInstance.nodeName);
NodeHandler nodeHandler = newInstance(nodeDef.getHandlerClass());
try {
nodeHandler.kill(new Context(nodeDef, node, null));
killedNodes.add(nodeDef.getName());
} catch (Exception ex) {
log.warn(XLog.STD, "Error killing node [{0}]", nodeDef.getName(), ex);
}
}
}
return killedNodes;
}
public LiteWorkflowApp getProcessDefinition() {
return def;
}
private static String createChildPath(String path, String child) {
return path + child + PATH_SEPARATOR;
}
private static String getParentPath(String path) {
path = path.substring(0, path.length() - 1);
return (path.length() == 0) ? null : path.substring(0, path.lastIndexOf(PATH_SEPARATOR) + 1);
}
private static String createFullTransition(String executionPath, String transition) {
return executionPath + TRANSITION_SEPARATOR + transition;
}
private static String getExecutionPath(String fullTransition) {
int index = fullTransition.indexOf(TRANSITION_SEPARATOR);
if (index == -1) {
throw new IllegalArgumentException("Invalid fullTransition");
}
return fullTransition.substring(0, index);
}
private static String getTransitionNode(String fullTransition) {
int index = fullTransition.indexOf(TRANSITION_SEPARATOR);
if (index == -1) {
throw new IllegalArgumentException("Invalid fullTransition");
}
return fullTransition.substring(index + 1);
}
private NodeHandler newInstance(Class<? extends NodeHandler> handler) {
return (NodeHandler) ReflectionUtils.newInstance(handler, null);
}
private void refreshLog() {
XLog.Info.get().setParameter(XLogService.USER, conf.get(OozieClient.USER_NAME));
XLog.Info.get().setParameter(XLogService.GROUP, conf.get(OozieClient.GROUP_NAME));
XLog.Info.get().setParameter(DagXLogInfoService.APP, def.getName());
XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN, ""));
XLog.Info.get().setParameter(DagXLogInfoService.JOB, instanceId);
log = XLog.getLog(getClass());
}
public Status getStatus() {
return status;
}
public void setStatus(Status status) {
this.status = status;
}
@Override
public void write(DataOutput dOut) throws IOException {
dOut.writeUTF(instanceId);
// Hadoop Configuration has to get its act right
ByteArrayOutputStream baos = new ByteArrayOutputStream();
conf.writeXml(baos);
baos.close();
byte[] array = baos.toByteArray();
dOut.writeInt(array.length);
dOut.write(array);
def.write(dOut);
dOut.writeUTF(status.toString());
dOut.writeInt(executionPaths.size());
for (Map.Entry<String, NodeInstance> entry : executionPaths.entrySet()) {
dOut.writeUTF(entry.getKey());
dOut.writeUTF(entry.getValue().nodeName);
dOut.writeBoolean(entry.getValue().started);
}
dOut.writeInt(persistentVars.size());
for (Map.Entry<String, String> entry : persistentVars.entrySet()) {
dOut.writeUTF(entry.getKey());
writeStringAsBytes(entry.getValue(), dOut);
}
}
@Override
public void readFields(DataInput dIn) throws IOException {
instanceId = dIn.readUTF();
// Hadoop Configuration has to get its act right
int len = dIn.readInt();
byte[] array = new byte[len];
dIn.readFully(array);
ByteArrayInputStream bais = new ByteArrayInputStream(array);
conf = new XConfiguration(bais);
def = new LiteWorkflowApp();
def.readFields(dIn);
status = Status.valueOf(dIn.readUTF());
int numExPaths = dIn.readInt();
for (int x = 0; x < numExPaths; x++) {
String path = dIn.readUTF();
String nodeName = dIn.readUTF();
boolean isStarted = dIn.readBoolean();
NodeInstance nodeInstance = new NodeInstance(nodeName);
nodeInstance.started = isStarted;
executionPaths.put(path, nodeInstance);
}
int numVars = dIn.readInt();
for (int x = 0; x < numVars; x++) {
String vName = dIn.readUTF();
String vVal = readBytesAsString(dIn);
persistentVars.put(vName, vVal);
}
refreshLog();
}
private void writeStringAsBytes(String value, DataOutput dOut) throws IOException {
if (value == null) {
dOut.writeUTF(null);
return;
}
dOut.writeUTF(DATA_VERSION);
byte[] data = value.getBytes("UTF-8");
dOut.writeInt(data.length);
dOut.write(data);
}
private String readBytesAsString(DataInput dIn) throws IOException {
String value = dIn.readUTF();
if (value != null && value.equals(DATA_VERSION)) {
int length = dIn.readInt();
byte[] data = new byte[length];
dIn.readFully(data);
value = new String(data, "UTF-8");
}
return value;
}
@Override
public Configuration getConf() {
return conf;
}
@Override
public WorkflowApp getApp() {
return def;
}
@Override
public String getId() {
return instanceId;
}
@Override
public String getTransition(String node) {
return persistentVars.get(node + WorkflowInstance.NODE_VAR_SEPARATOR + TRANSITION_TO);
}
@Override
public boolean equals(Object o) {
return (o != null)
&& (getClass().isInstance(o))
&& ((WorkflowInstance) o).getId().equals(instanceId);
}
@Override
public int hashCode() {
return instanceId.hashCode();
}
}
/**
* @param offset
* @return n-th available latest instance Date-Time for SYNC data-set
* @throws Exception
*/
private static String coord_latest_sync(int offset) throws Exception {
if (offset > 0) {
throw new RuntimeException(
"For latest there is no meaning " + "of positive instance. n should be <=0" + offset);
}
ELEvaluator eval = ELEvaluator.getCurrent();
String retVal = "";
int datasetFrequency = (int) getDSFrequency(); // in minutes
TimeUnit dsTimeUnit = getDSTimeUnit();
int[] instCount = new int[1];
Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
if (nominalInstanceCal != null) {
Calendar initInstance = getInitialInstanceCal();
SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
if (ds == null) {
throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
}
String uriTemplate = ds.getUriTemplate();
Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
if (conf == null) {
throw new RuntimeException(
"Associated Configuration should be defined with key " + CONFIGURATION);
}
int available = 0;
boolean resolved = false;
String user =
ParamChecker.notEmpty(
(String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
String group =
ParamChecker.notEmpty(
(String) eval.getVariable(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
String doneFlag = ds.getDoneFlag();
while (nominalInstanceCal.compareTo(initInstance) >= 0) {
ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
String uriPath = uriEval.evaluate(uriTemplate, String.class);
String pathWithDoneFlag = uriPath;
if (doneFlag.length() > 0) {
pathWithDoneFlag += "/" + doneFlag;
}
if (isPathAvailable(pathWithDoneFlag, user, group, conf)) {
XLog.getLog(CoordELFunctions.class)
.debug("Found latest(" + available + "): " + pathWithDoneFlag);
if (available == offset) {
XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag);
resolved = true;
retVal = DateUtils.formatDateUTC(nominalInstanceCal);
eval.setVariable("resolved_path", uriPath);
break;
}
available--;
}
// nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(),
// -datasetFrequency);
nominalInstanceCal = (Calendar) initInstance.clone();
instCount[0]--;
nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
// DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
}
if (!resolved) {
// return unchanged latest function with variable 'is_resolved'
// to 'false'
eval.setVariable("is_resolved", Boolean.FALSE);
retVal = "${coord:latest(" + offset + ")}";
} else {
eval.setVariable("is_resolved", Boolean.TRUE);
}
} else { // No feasible nominal time
eval.setVariable("is_resolved", Boolean.FALSE);
}
return retVal;
}
/** The authorization service provides all authorization checks. */
public class AuthorizationService implements Service {
public static final String CONF_PREFIX = Service.CONF_PREFIX + "AuthorizationService.";
/** Configuration parameter to enable or disable Oozie admin role. */
public static final String CONF_SECURITY_ENABLED = CONF_PREFIX + "security.enabled";
/** Configuration parameter to enable or disable Oozie admin role. */
public static final String CONF_AUTHORIZATION_ENABLED = CONF_PREFIX + "authorization.enabled";
/** Configuration parameter to enable old behavior default group as ACL. */
public static final String CONF_DEFAULT_GROUP_AS_ACL = CONF_PREFIX + "default.group.as.acl";
/**
* Configuration parameter to define admin groups, if NULL/empty the adminusers.txt file is used.
*/
public static final String CONF_ADMIN_GROUPS = CONF_PREFIX + "admin.groups";
/** File that contains list of admin users for Oozie. */
public static final String ADMIN_USERS_FILE = "adminusers.txt";
protected static final String INSTRUMENTATION_GROUP = "authorization";
protected static final String INSTR_FAILED_AUTH_COUNTER = "authorization.failed";
private Set<String> adminGroups;
private Set<String> adminUsers;
private boolean authorizationEnabled;
private boolean useDefaultGroupAsAcl;
private final XLog log = XLog.getLog(getClass());
private Instrumentation instrumentation;
private String[] getTrimmedStrings(String str) {
if (null == str || "".equals(str.trim())) {
return new String[0];
}
return str.trim().split("\\s*,\\s*");
}
/**
* Initialize the service.
*
* <p>Reads the security related configuration. parameters - security enabled and list of super
* users.
*
* @param services services instance.
* @throws ServiceException thrown if the service could not be initialized.
*/
public void init(Services services) throws ServiceException {
authorizationEnabled =
ConfigUtils.getWithDeprecatedCheck(
services.getConf(), CONF_AUTHORIZATION_ENABLED, CONF_SECURITY_ENABLED, false);
if (authorizationEnabled) {
log.info("Oozie running with authorization enabled");
useDefaultGroupAsAcl = Services.get().getConf().getBoolean(CONF_DEFAULT_GROUP_AS_ACL, false);
String[] str = getTrimmedStrings(Services.get().getConf().get(CONF_ADMIN_GROUPS));
if (str.length > 0) {
log.info("Admin users will be checked against the defined admin groups");
adminGroups = new HashSet<String>();
for (String s : str) {
adminGroups.add(s.trim());
}
} else {
log.info("Admin users will be checked against the 'adminusers.txt' file contents");
adminUsers = new HashSet<String>();
loadAdminUsers();
}
} else {
log.warn("Oozie running with authorization disabled");
}
instrumentation = Services.get().get(InstrumentationService.class).get();
}
/**
* Return if security is enabled or not.
*
* @return if security is enabled or not.
*/
@Deprecated
public boolean isSecurityEnabled() {
return authorizationEnabled;
}
public boolean useDefaultGroupAsAcl() {
return useDefaultGroupAsAcl;
}
/**
* Return if security is enabled or not.
*
* @return if security is enabled or not.
*/
public boolean isAuthorizationEnabled() {
return isSecurityEnabled();
}
/**
* Load the list of admin users from {@link AuthorizationService#ADMIN_USERS_FILE}
*
* @throws ServiceException if the admin user list could not be loaded.
*/
private void loadAdminUsers() throws ServiceException {
String configDir = Services.get().get(ConfigurationService.class).getConfigDir();
if (configDir != null) {
File file = new File(configDir, ADMIN_USERS_FILE);
if (file.exists()) {
try {
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file)));
try {
String line = br.readLine();
while (line != null) {
line = line.trim();
if (line.length() > 0 && !line.startsWith("#")) {
adminUsers.add(line);
}
line = br.readLine();
}
} catch (IOException ex) {
throw new ServiceException(ErrorCode.E0160, file.getAbsolutePath(), ex);
}
} catch (FileNotFoundException ex) {
throw new ServiceException(ErrorCode.E0160, ex);
}
} else {
log.warn(
"Admin users file not available in config dir [{0}], running without admin users",
configDir);
}
} else {
log.warn("Reading configuration from classpath, running without admin users");
}
}
/**
* Destroy the service.
*
* <p>This implementation does a NOP.
*/
public void destroy() {}
/**
* Return the public interface of the service.
*
* @return {@link AuthorizationService}.
*/
public Class<? extends Service> getInterface() {
return AuthorizationService.class;
}
/**
* Check if the user belongs to the group or not.
*
* @param user user name.
* @param group group name.
* @return if the user belongs to the group or not.
* @throws AuthorizationException thrown if the authorization query can not be performed.
*/
protected boolean isUserInGroup(String user, String group) throws AuthorizationException {
GroupsService groupsService = Services.get().get(GroupsService.class);
try {
return groupsService.getGroups(user).contains(group);
} catch (IOException ex) {
throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex);
}
}
/**
* Check if the user belongs to the group or not.
*
* <p>
*
* <p>Subclasses should override the {@link #isUserInGroup} method.
*
* @param user user name.
* @param group group name.
* @throws AuthorizationException thrown if the user is not authorized for the group or if the
* authorization query can not be performed.
*/
public void authorizeForGroup(String user, String group) throws AuthorizationException {
if (authorizationEnabled && !isUserInGroup(user, group)) {
throw new AuthorizationException(ErrorCode.E0502, user, group);
}
}
/**
* Return the default group to which the user belongs.
*
* <p>This implementation always returns 'users'.
*
* @param user user name.
* @return default group of user.
* @throws AuthorizationException thrown if the default group con not be retrieved.
*/
public String getDefaultGroup(String user) throws AuthorizationException {
try {
return Services.get().get(GroupsService.class).getGroups(user).get(0);
} catch (IOException ex) {
throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex);
}
}
/**
* Check if the user has admin privileges.
*
* <p>If admin is disabled it returns always <code>true</code>.
*
* <p>If admin is enabled it returns <code>true</code> if the user is in the <code>adminusers.txt
* </code> file.
*
* @param user user name.
* @return if the user has admin privileges or not.
*/
protected boolean isAdmin(String user) {
boolean admin = false;
if (adminUsers != null) {
admin = adminUsers.contains(user);
} else {
for (String adminGroup : adminGroups) {
try {
admin = isUserInGroup(user, adminGroup);
if (admin) {
break;
}
} catch (AuthorizationException ex) {
log.warn("Admin check failed, " + ex.toString(), ex);
break;
}
}
}
return admin;
}
/**
* Check if the user has admin privileges.
*
* <p>Subclasses should override the {@link #isUserInGroup} method.
*
* @param user user name.
* @param write indicates if the check is for read or write admin tasks (in this implementation
* this is ignored)
* @throws AuthorizationException thrown if user does not have admin priviledges.
*/
public void authorizeForAdmin(String user, boolean write) throws AuthorizationException {
if (authorizationEnabled && write && !isAdmin(user)) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0503, user);
}
}
/**
* Check if the user+group is authorized to use the specified application.
*
* <p>The check is done by checking the file system permissions on the workflow application.
*
* @param user user name.
* @param group group name.
* @param appPath application path.
* @throws AuthorizationException thrown if the user is not authorized for the app.
*/
public void authorizeForApp(String user, String group, String appPath, Configuration jobConf)
throws AuthorizationException {
try {
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
URI uri = new Path(appPath).toUri();
Configuration fsConf = has.createJobConf(uri.getAuthority());
FileSystem fs = has.createFileSystem(user, uri, fsConf);
Path path = new Path(appPath);
try {
if (!fs.exists(path)) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0504, appPath);
}
Path wfXml = new Path(path, "workflow.xml");
if (!fs.exists(wfXml)) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0505, appPath);
}
if (!fs.isFile(wfXml)) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0506, appPath);
}
fs.open(wfXml).close();
}
// TODO change this when stopping support of 0.18 to the new
// Exception
catch (org.apache.hadoop.fs.permission.AccessControlException ex) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0507, appPath, ex.getMessage(), ex);
}
} catch (IOException ex) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex);
} catch (HadoopAccessorException e) {
throw new AuthorizationException(e);
}
}
/**
* Check if the user+group is authorized to use the specified application.
*
* <p>The check is done by checking the file system permissions on the workflow application.
*
* @param user user name.
* @param group group name.
* @param appPath application path.
* @param fileName workflow or coordinator.xml
* @param conf
* @throws AuthorizationException thrown if the user is not authorized for the app.
*/
public void authorizeForApp(
String user, String group, String appPath, String fileName, Configuration conf)
throws AuthorizationException {
try {
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
URI uri = new Path(appPath).toUri();
Configuration fsConf = has.createJobConf(uri.getAuthority());
FileSystem fs = has.createFileSystem(user, uri, fsConf);
Path path = new Path(appPath);
try {
if (!fs.exists(path)) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0504, appPath);
}
if (conf.get(XOozieClient.IS_PROXY_SUBMISSION)
== null) { // Only further check existence of job definition files for non proxy
// submission jobs;
if (!fs.isFile(path)) {
Path appXml = new Path(path, fileName);
if (!fs.exists(appXml)) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0505, appPath);
}
if (!fs.isFile(appXml)) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0506, appPath);
}
fs.open(appXml).close();
}
}
}
// TODO change this when stopping support of 0.18 to the new
// Exception
catch (org.apache.hadoop.fs.permission.AccessControlException ex) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0507, appPath, ex.getMessage(), ex);
}
} catch (IOException ex) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex);
} catch (HadoopAccessorException e) {
throw new AuthorizationException(e);
}
}
private boolean isUserInAcl(String user, String aclStr) throws IOException {
boolean userInAcl = false;
if (aclStr != null && aclStr.trim().length() > 0) {
GroupsService groupsService = Services.get().get(GroupsService.class);
String[] acl = aclStr.split(",");
for (int i = 0; !userInAcl && i < acl.length; i++) {
String aclItem = acl[i].trim();
userInAcl = aclItem.equals(user) || groupsService.getGroups(user).equals(aclItem);
}
}
return userInAcl;
}
/**
* Check if the user+group is authorized to operate on the specified job.
*
* <p>Checks if the user is a super-user or the one who started the job.
*
* <p>Read operations are allowed to all users.
*
* @param user user name.
* @param jobId job id.
* @param write indicates if the check is for read or write job tasks.
* @throws AuthorizationException thrown if the user is not authorized for the job.
*/
public void authorizeForJob(String user, String jobId, boolean write)
throws AuthorizationException {
if (authorizationEnabled && write && !isAdmin(user)) {
try {
// handle workflow jobs
if (jobId.endsWith("-W")) {
WorkflowJobBean jobBean = null;
JPAService jpaService = Services.get().get(JPAService.class);
if (jpaService != null) {
try {
jobBean = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
} catch (JPAExecutorException je) {
throw new AuthorizationException(je);
}
} else {
throw new AuthorizationException(ErrorCode.E0610);
}
if (jobBean != null && !jobBean.getUser().equals(user)) {
if (!isUserInAcl(user, jobBean.getGroup())) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0508, user, jobId);
}
}
}
// handle bundle jobs
else if (jobId.endsWith("-B")) {
BundleJobBean jobBean = null;
JPAService jpaService = Services.get().get(JPAService.class);
if (jpaService != null) {
try {
jobBean = jpaService.execute(new BundleJobGetJPAExecutor(jobId));
} catch (JPAExecutorException je) {
throw new AuthorizationException(je);
}
} else {
throw new AuthorizationException(ErrorCode.E0610);
}
if (jobBean != null && !jobBean.getUser().equals(user)) {
if (!isUserInAcl(user, jobBean.getGroup())) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0509, user, jobId);
}
}
}
// handle coordinator jobs
else {
CoordinatorJobBean jobBean = null;
JPAService jpaService = Services.get().get(JPAService.class);
if (jpaService != null) {
try {
jobBean = jpaService.execute(new CoordJobGetJPAExecutor(jobId));
} catch (JPAExecutorException je) {
throw new AuthorizationException(je);
}
} else {
throw new AuthorizationException(ErrorCode.E0610);
}
if (jobBean != null && !jobBean.getUser().equals(user)) {
if (!isUserInAcl(user, jobBean.getGroup())) {
incrCounter(INSTR_FAILED_AUTH_COUNTER, 1);
throw new AuthorizationException(ErrorCode.E0509, user, jobId);
}
}
}
} catch (IOException ex) {
throw new AuthorizationException(ErrorCode.E0501, ex.getMessage(), ex);
}
}
}
/**
* Convenience method for instrumentation counters.
*
* @param name counter name.
* @param count count to increment the counter.
*/
private void incrCounter(String name, int count) {
if (instrumentation != null) {
instrumentation.incr(INSTRUMENTATION_GROUP, name, count);
}
}
}
/**
* Built in service that initializes the services configuration.
*
* <p>The configuration loading sequence is identical to Hadoop configuration loading sequence.
*
* <p>Default values are loaded from the 'oozie-default.xml' file from the classpath, then site
* configured values are loaded from a site configuration file from the Oozie configuration
* directory.
*
* <p>The Oozie configuration directory is resolved using the <code>OOZIE_HOME</code> environment
* variable as <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME</code> environment variable
* is not defined the initialization of the <code>ConfigurationService</code> fails.
*
* <p>The site configuration is loaded from the <code>oozie-site.xml</code> file in the
* configuration directory.
*
* <p>The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE
* </code> environment variable to an alternate file name. The alternate file must ber in the Oozie
* configuration directory.
*
* <p>Configuration properties, prefixed with 'oozie.', passed as system properties overrides
* default and site values.
*
* <p>The configuration service logs details on how the configuration was loaded as well as what
* properties were overrode via system properties settings.
*/
public class ConfigurationService implements Service, Instrumentable {
private static final String INSTRUMENTATION_GROUP = "configuration";
public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
public static final String CONF_VERIFY_AVAILABLE_PROPS =
CONF_PREFIX + "verify.available.properties";
/** System property that indicates the configuration directory. */
public static final String OOZIE_CONFIG_DIR = "oozie.config.dir";
/** System property that indicates the data directory. */
public static final String OOZIE_DATA_DIR = "oozie.data.dir";
/** System property that indicates the name of the site configuration file to load. */
public static final String OOZIE_CONFIG_FILE = "oozie.config.file";
private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
private static final Set<String> CONF_SYS_PROPS = new HashSet<String>();
private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
private static final Set<String> MASK_PROPS = new HashSet<String>();
private static Map<String, String> defaultConfigs = new HashMap<String, String>();
private static Method getPasswordMethod;
static {
// all this properties are seeded as system properties, no need to log changes
IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR);
IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR);
IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE);
IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR);
IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR);
IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE);
IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD);
CONF_SYS_PROPS.add("oozie.http.hostname");
CONF_SYS_PROPS.add("oozie.http.port");
CONF_SYS_PROPS.add(ZKUtils.OOZIE_INSTANCE_ID);
// These properties should be masked when displayed because they contain sensitive info (e.g.
// password)
MASK_PROPS.add(JPAService.CONF_PASSWORD);
MASK_PROPS.add("oozie.authentication.signature.secret");
try {
// Only supported in Hadoop 2.6.0+
getPasswordMethod = Configuration.class.getMethod("getPassword", String.class);
} catch (NoSuchMethodException e) {
// Not supported
getPasswordMethod = null;
}
}
public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
public static final String SITE_CONFIG_FILE = "oozie-site.xml";
private static XLog log = XLog.getLog(ConfigurationService.class);
private String configDir;
private String configFile;
private LogChangesConfiguration configuration;
public ConfigurationService() {
log = XLog.getLog(ConfigurationService.class);
}
/**
* Initialize the log service.
*
* @param services services instance.
* @throws ServiceException thrown if the log service could not be initialized.
*/
public void init(Services services) throws ServiceException {
configDir = getConfigurationDirectory();
configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE);
if (configFile.contains("/")) {
throw new ServiceException(ErrorCode.E0022, configFile);
}
log.info("Oozie home dir [{0}]", Services.getOozieHome());
log.info("Oozie conf dir [{0}]", configDir);
log.info("Oozie conf file [{0}]", configFile);
configFile = new File(configDir, configFile).toString();
configuration = loadConf();
if (configuration.getBoolean(CONF_VERIFY_AVAILABLE_PROPS, false)) {
verifyConfigurationName();
}
}
public static String getConfigurationDirectory() throws ServiceException {
String oozieHome = Services.getOozieHome();
String configDir = System.getProperty(OOZIE_CONFIG_DIR, null);
File file = configDir == null ? new File(oozieHome, "conf") : new File(configDir);
if (!file.exists()) {
throw new ServiceException(ErrorCode.E0024, configDir);
}
return file.getPath();
}
/** Destroy the configuration service. */
public void destroy() {
configuration = null;
}
/**
* Return the public interface for configuration service.
*
* @return {@link ConfigurationService}.
*/
public Class<? extends Service> getInterface() {
return ConfigurationService.class;
}
/**
* Return the services configuration.
*
* @return the services configuration.
*/
public Configuration getConf() {
if (configuration == null) {
throw new IllegalStateException("Not initialized");
}
return configuration;
}
/**
* Return Oozie configuration directory.
*
* @return Oozie configuration directory.
*/
public String getConfigDir() {
return configDir;
}
private InputStream getDefaultConfiguration() throws ServiceException, IOException {
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
if (inputStream == null) {
throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
}
return inputStream;
}
private LogChangesConfiguration loadConf() throws ServiceException {
XConfiguration configuration;
try {
InputStream inputStream = getDefaultConfiguration();
configuration = loadConfig(inputStream, true);
File file = new File(configFile);
if (!file.exists()) {
log.info("Missing site configuration file [{0}]", configFile);
} else {
inputStream = new FileInputStream(configFile);
XConfiguration siteConfiguration = loadConfig(inputStream, false);
XConfiguration.injectDefaults(configuration, siteConfiguration);
configuration = siteConfiguration;
}
} catch (IOException ex) {
throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
}
if (log.isTraceEnabled()) {
try {
StringWriter writer = new StringWriter();
for (Map.Entry<String, String> entry : configuration) {
String value = getValue(configuration, entry.getKey());
writer.write(" " + entry.getKey() + " = " + value + "\n");
}
writer.close();
log.trace("Configuration:\n{0}---", writer.toString());
} catch (IOException ex) {
throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
}
}
String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
if (ignoreSysProps != null) {
IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
}
for (Map.Entry<String, String> entry : configuration) {
String sysValue = System.getProperty(entry.getKey());
if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
configuration.set(entry.getKey(), sysValue);
}
}
for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
String name = (String) entry.getKey();
if (!IGNORE_SYS_PROPS.contains(name)) {
if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
if (configuration.get(name) == null) {
log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
}
}
}
}
// Backward compatible, we should still support -Dparam.
for (String key : CONF_SYS_PROPS) {
String sysValue = System.getProperty(key);
if (sysValue != null && !IGNORE_SYS_PROPS.contains(key)) {
log.info(
"Overriding configuration with system property. Key [{0}], Value [{1}] ",
key, sysValue);
configuration.set(key, sysValue);
}
}
return new LogChangesConfiguration(configuration);
}
private XConfiguration loadConfig(InputStream inputStream, boolean defaultConfig)
throws IOException, ServiceException {
XConfiguration configuration;
configuration = new XConfiguration(inputStream);
for (Map.Entry<String, String> entry : configuration) {
if (defaultConfig) {
defaultConfigs.put(entry.getKey(), entry.getValue());
} else {
log.debug("Overriding configuration with oozie-site, [{0}]", entry.getKey());
}
}
return configuration;
}
private class LogChangesConfiguration extends XConfiguration {
public LogChangesConfiguration(Configuration conf) {
for (Map.Entry<String, String> entry : conf) {
if (get(entry.getKey()) == null) {
setValue(entry.getKey(), entry.getValue());
}
}
}
public String[] getStrings(String name) {
String s = get(name);
return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
}
public String[] getStrings(String name, String[] defaultValue) {
String s = get(name);
if (s == null) {
log.debug(
XLog.OPS,
"Configuration property [{0}] not found, use given value [{1}]",
name,
Arrays.asList(defaultValue).toString());
}
return (s != null && s.trim().length() > 0) ? super.getStrings(name) : defaultValue;
}
public String get(String name, String defaultValue) {
String value = get(name);
if (value == null) {
boolean maskValue = MASK_PROPS.contains(name);
value = defaultValue;
String logValue = (maskValue) ? "**MASKED**" : defaultValue;
log.debug(
XLog.OPS,
"Configuration property [{0}] not found, use given value [{1}]",
name,
logValue);
}
return value;
}
public void set(String name, String value) {
setValue(name, value);
boolean maskValue = MASK_PROPS.contains(name);
value = (maskValue) ? "**MASKED**" : value;
log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
}
public boolean getBoolean(String name, boolean defaultValue) {
String value = get(name);
if (value == null) {
log.debug(
XLog.OPS,
"Configuration property [{0}] not found, use given value [{1}]",
name,
defaultValue);
}
return super.getBoolean(name, defaultValue);
}
public int getInt(String name, int defaultValue) {
String value = get(name);
if (value == null) {
log.debug(
XLog.OPS,
"Configuration property [{0}] not found, use given value [{1}]",
name,
defaultValue);
}
return super.getInt(name, defaultValue);
}
public long getLong(String name, long defaultValue) {
String value = get(name);
if (value == null) {
log.debug(
XLog.OPS,
"Configuration property [{0}] not found, use given value [{1}]",
name,
defaultValue);
}
return super.getLong(name, defaultValue);
}
public float getFloat(String name, float defaultValue) {
String value = get(name);
if (value == null) {
log.debug(
XLog.OPS,
"Configuration property [{0}] not found, use given value [{1}]",
name,
defaultValue);
}
return super.getFloat(name, defaultValue);
}
public Class<?>[] getClasses(String name, Class<?>... defaultValue) {
String value = get(name);
if (value == null) {
log.debug(
XLog.OPS,
"Configuration property [{0}] not found, use given value [{1}]",
name,
defaultValue);
}
return super.getClasses(name, defaultValue);
}
public Class<?> getClass(String name, Class<?> defaultValue) {
String value = get(name);
if (value == null) {
log.debug(
XLog.OPS,
"Configuration property [{0}] not found, use given value [{1}]",
name,
defaultValue);
return defaultValue;
}
try {
return getClassByName(value);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
private void setValue(String name, String value) {
super.set(name, value);
}
}
/**
* Instruments the configuration service.
*
* <p>It sets instrumentation variables indicating the config dir and config file used.
*
* @param instr instrumentation to use.
*/
public void instrument(Instrumentation instr) {
instr.addVariable(
INSTRUMENTATION_GROUP,
"config.dir",
new Instrumentation.Variable<String>() {
public String getValue() {
return configDir;
}
});
instr.addVariable(
INSTRUMENTATION_GROUP,
"config.file",
new Instrumentation.Variable<String>() {
public String getValue() {
return configFile;
}
});
}
/**
* Return a configuration with all sensitive values masked.
*
* @return masked configuration.
*/
public Configuration getMaskedConfiguration() {
XConfiguration maskedConf = new XConfiguration();
Configuration conf = getConf();
for (Map.Entry<String, String> entry : conf) {
String name = entry.getKey();
String value = getValue(conf, name);
maskedConf.set(name, value);
}
return maskedConf;
}
private String getValue(Configuration config, String key) {
String value;
if (MASK_PROPS.contains(key)) {
value = "**MASKED**";
} else {
value = config.get(key);
}
return value;
}
/**
* Gets the oozie configuration value in oozie-default.
*
* @param name
* @return the configuration value of the <code>name</code> otherwise null
*/
private String getDefaultOozieConfig(String name) {
return defaultConfigs.get(name);
}
/** Verify the configuration is in oozie-default */
public void verifyConfigurationName() {
for (Map.Entry<String, String> entry : configuration) {
if (getDefaultOozieConfig(entry.getKey()) == null) {
log.warn("Invalid configuration defined, [{0}] ", entry.getKey());
}
}
}
@VisibleForTesting
public static void set(String name, String value) {
Configuration conf = Services.get().getConf();
conf.set(name, value);
}
@VisibleForTesting
public static void setBoolean(String name, boolean value) {
Configuration conf = Services.get().getConf();
conf.setBoolean(name, value);
}
public static String get(String name) {
Configuration conf = Services.get().getConf();
return get(conf, name);
}
public static String get(Configuration conf, String name) {
return conf.get(name, ConfigUtils.STRING_DEFAULT);
}
public static String[] getStrings(String name) {
Configuration conf = Services.get().getConf();
return getStrings(conf, name);
}
public static String[] getStrings(Configuration conf, String name) {
return conf.getStrings(name, new String[0]);
}
public static boolean getBoolean(String name) {
Configuration conf = Services.get().getConf();
return getBoolean(conf, name);
}
public static boolean getBoolean(Configuration conf, String name) {
return conf.getBoolean(name, ConfigUtils.BOOLEAN_DEFAULT);
}
public static int getInt(String name) {
Configuration conf = Services.get().getConf();
return getInt(conf, name);
}
public static int getInt(Configuration conf, String name) {
return conf.getInt(name, ConfigUtils.INT_DEFAULT);
}
public static float getFloat(String name) {
Configuration conf = Services.get().getConf();
return conf.getFloat(name, ConfigUtils.FLOAT_DEFAULT);
}
public static long getLong(String name) {
Configuration conf = Services.get().getConf();
return getLong(conf, name);
}
public static long getLong(Configuration conf, String name) {
return conf.getLong(name, ConfigUtils.LONG_DEFAULT);
}
public static Class<?>[] getClasses(String name) {
Configuration conf = Services.get().getConf();
return getClasses(conf, name);
}
public static Class<?>[] getClasses(Configuration conf, String name) {
return conf.getClasses(name);
}
public static Class<?> getClass(Configuration conf, String name) {
return conf.getClass(name, Object.class);
}
public static String getPassword(Configuration conf, String name) {
if (getPasswordMethod != null) {
try {
char[] pass = (char[]) getPasswordMethod.invoke(conf, name);
return new String(pass);
} catch (IllegalAccessException e) {
log.error(e);
throw new IllegalArgumentException("Could not load password for [" + name + "]", e);
} catch (InvocationTargetException e) {
log.error(e);
throw new IllegalArgumentException("Could not load password for [" + name + "]", e);
}
} else {
return conf.get(name);
}
}
public static String getPassword(String name) {
Configuration conf = Services.get().getConf();
return getPassword(conf, name);
}
}
public ConfigurationService() {
log = XLog.getLog(ConfigurationService.class);
}
/**
* The ELService creates {@link ELEvaluator} instances preconfigured with constants and functions
* defined in the configuration.
*
* <p>The following configuration parameters control the EL service:
*
* <p>{@link #CONF_CONSTANTS} list of constant definitions to be available for EL evaluations.
*
* <p>{@link #CONF_FUNCTIONS} list of function definitions to be available for EL evalations.
*
* <p>Definitions must be separated by a comma, definitions are trimmed.
*
* <p>The syntax for a constant definition is <code>PREFIX:NAME=CLASS_NAME#CONSTANT_NAME</code>.
*
* <p>The syntax for a constant definition is <code>PREFIX:NAME=CLASS_NAME#METHOD_NAME</code>.
*/
public class ELService implements Service {
public static final String CONF_PREFIX = Service.CONF_PREFIX + "ELService.";
public static final String CONF_CONSTANTS = CONF_PREFIX + "constants.";
public static final String CONF_EXT_CONSTANTS = CONF_PREFIX + "ext.constants.";
public static final String CONF_FUNCTIONS = CONF_PREFIX + "functions.";
public static final String CONF_EXT_FUNCTIONS = CONF_PREFIX + "ext.functions.";
public static final String CONF_GROUPS = CONF_PREFIX + "groups";
private final XLog log = XLog.getLog(getClass());
// <Group Name>, <List of constants>
private HashMap<String, List<ELConstant>> constants;
// <Group Name>, <List of functions>
private HashMap<String, List<ELFunction>> functions;
private static class ELConstant {
private String name;
private Object value;
private ELConstant(String prefix, String name, Object value) {
if (prefix.length() > 0) {
name = prefix + ":" + name;
}
this.name = name;
this.value = value;
}
}
private static class ELFunction {
private String prefix;
private String name;
private Method method;
private ELFunction(String prefix, String name, Method method) {
this.prefix = prefix;
this.name = name;
this.method = method;
}
}
private List<ELService.ELConstant> extractConstants(Configuration conf, String key)
throws ServiceException {
List<ELService.ELConstant> list = new ArrayList<ELService.ELConstant>();
if (conf.get(key, "").trim().length() > 0) {
for (String function : ConfigurationService.getStrings(conf, key)) {
String[] parts = parseDefinition(function);
list.add(new ELConstant(parts[0], parts[1], findConstant(parts[2], parts[3])));
log.trace("Registered prefix:constant[{0}:{1}] for class#field[{2}#{3}]", (Object[]) parts);
}
}
return list;
}
private List<ELService.ELFunction> extractFunctions(Configuration conf, String key)
throws ServiceException {
List<ELService.ELFunction> list = new ArrayList<ELService.ELFunction>();
if (conf.get(key, "").trim().length() > 0) {
for (String function : ConfigurationService.getStrings(conf, key)) {
String[] parts = parseDefinition(function);
list.add(new ELFunction(parts[0], parts[1], findMethod(parts[2], parts[3])));
log.trace("Registered prefix:constant[{0}:{1}] for class#field[{2}#{3}]", (Object[]) parts);
}
}
return list;
}
/**
* Initialize the EL service.
*
* @param services services instance.
* @throws ServiceException thrown if the EL service could not be initialized.
*/
@Override
public synchronized void init(Services services) throws ServiceException {
log.trace("Constants and functions registration");
constants = new HashMap<String, List<ELConstant>>();
functions = new HashMap<String, List<ELFunction>>();
// Get the list of group names from configuration file
// defined in the property tag: oozie.service.ELSerice.groups
// String []groupList = services.getConf().get(CONF_GROUPS, "").trim().split(",");
String[] groupList = ConfigurationService.getStrings(services.getConf(), CONF_GROUPS);
// For each group, collect the required functions and constants
// and store it into HashMap
for (String group : groupList) {
List<ELConstant> tmpConstants = new ArrayList<ELConstant>();
tmpConstants.addAll(extractConstants(services.getConf(), CONF_CONSTANTS + group));
tmpConstants.addAll(extractConstants(services.getConf(), CONF_EXT_CONSTANTS + group));
constants.put(group, tmpConstants);
List<ELFunction> tmpFunctions = new ArrayList<ELFunction>();
tmpFunctions.addAll(extractFunctions(services.getConf(), CONF_FUNCTIONS + group));
tmpFunctions.addAll(extractFunctions(services.getConf(), CONF_EXT_FUNCTIONS + group));
functions.put(group, tmpFunctions);
}
}
/** Destroy the EL service. */
@Override
public void destroy() {
constants = null;
functions = null;
}
/**
* Return the public interface for EL service.
*
* @return {@link ELService}.
*/
@Override
public Class<? extends Service> getInterface() {
return ELService.class;
}
/**
* Return an {@link ELEvaluator} pre-configured with the constants and functions for the specific
* group of EL-functions and variables defined in the configuration. If the group name doesn't
* exist, IllegalArgumentException is thrown
*
* @param group: Name of the group of required EL Evaluator.
* @return a preconfigured {@link ELEvaluator}.
*/
public ELEvaluator createEvaluator(String group) {
ELEvaluator.Context context = new ELEvaluator.Context();
boolean groupDefined = false;
if (constants.containsKey(group)) {
for (ELConstant constant : constants.get(group)) {
context.setVariable(constant.name, constant.value);
}
groupDefined = true;
}
if (functions.containsKey(group)) {
for (ELFunction function : functions.get(group)) {
context.addFunction(function.prefix, function.name, function.method);
}
groupDefined = true;
}
if (groupDefined == false) {
throw new IllegalArgumentException("Group " + group + " is not defined");
}
return new ELEvaluator(context);
}
private static String[] parseDefinition(String str) throws ServiceException {
try {
str = str.trim();
if (!str.contains(":")) {
str = ":" + str;
}
String[] parts = str.split(":");
String prefix = parts[0];
parts = parts[1].split("=");
String name = parts[0];
parts = parts[1].split("#");
String klass = parts[0];
String method = parts[1];
return new String[] {prefix, name, klass, method};
} catch (Exception ex) {
throw new ServiceException(ErrorCode.E0110, str, ex.getMessage(), ex);
}
}
public static Method findMethod(String className, String methodName) throws ServiceException {
Method method = null;
try {
Class klass = Thread.currentThread().getContextClassLoader().loadClass(className);
for (Method m : klass.getMethods()) {
if (m.getName().equals(methodName)) {
method = m;
break;
}
}
if (method == null) {
throw new ServiceException(ErrorCode.E0111, className, methodName);
}
if ((method.getModifiers() & (Modifier.PUBLIC | Modifier.STATIC))
!= (Modifier.PUBLIC | Modifier.STATIC)) {
throw new ServiceException(ErrorCode.E0112, className, methodName);
}
} catch (ClassNotFoundException ex) {
throw new ServiceException(ErrorCode.E0113, className);
}
return method;
}
public static Object findConstant(String className, String constantName) throws ServiceException {
try {
Class klass = Thread.currentThread().getContextClassLoader().loadClass(className);
Field field = klass.getField(constantName);
if ((field.getModifiers() & (Modifier.PUBLIC | Modifier.STATIC))
!= (Modifier.PUBLIC | Modifier.STATIC)) {
throw new ServiceException(ErrorCode.E0114, className, constantName);
}
return field.get(null);
} catch (IllegalAccessException ex) {
throw new IllegalArgumentException(ex);
} catch (NoSuchFieldException ex) {
throw new ServiceException(ErrorCode.E0115, className, constantName);
} catch (ClassNotFoundException ex) {
throw new ServiceException(ErrorCode.E0113, className);
}
}
}
@Override
protected Void execute() throws CommandException {
LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
WorkflowInstance workflowInstance = wfJob.getWorkflowInstance();
workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob);
boolean completed = false;
boolean skipAction = false;
if (wfAction == null) {
if (wfJob.getStatus() == WorkflowJob.Status.PREP) {
try {
completed = workflowInstance.start();
} catch (WorkflowException e) {
throw new CommandException(e);
}
wfJob.setStatus(WorkflowJob.Status.RUNNING);
wfJob.setStartTime(new Date());
wfJob.setWorkflowInstance(workflowInstance);
// 1. Add SLA status event for WF-JOB with status STARTED
// 2. Add SLA registration events for all WF_ACTIONS
SLADbXOperations.writeStausEvent(
wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB);
writeSLARegistrationForAllActions(
workflowInstance.getApp().getDefinition(),
wfJob.getUser(),
wfJob.getGroup(),
wfJob.getConf());
queue(new NotificationXCommand(wfJob));
} else {
throw new CommandException(ErrorCode.E0801, wfJob.getId());
}
} else {
String skipVar =
workflowInstance.getVar(
wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
if (skipVar != null) {
skipAction = skipVar.equals("true");
}
try {
completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue());
} catch (WorkflowException e) {
throw new CommandException(e);
}
wfJob.setWorkflowInstance(workflowInstance);
wfAction.resetPending();
if (!skipAction) {
wfAction.setTransition(workflowInstance.getTransition(wfAction.getName()));
}
try {
jpaService.execute(new WorkflowActionUpdateJPAExecutor(wfAction));
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
}
if (completed) {
try {
for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) {
WorkflowActionBean actionToKill;
actionToKill = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToKillId));
actionToKill.setPending();
actionToKill.setStatus(WorkflowActionBean.Status.KILLED);
jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToKill));
queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType()));
}
for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) {
WorkflowActionBean actionToFail =
jpaService.execute(new WorkflowActionGetJPAExecutor(actionToFailId));
actionToFail.resetPending();
actionToFail.setStatus(WorkflowActionBean.Status.FAILED);
SLADbXOperations.writeStausEvent(
wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION);
jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToFail));
}
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
wfJob.setEndTime(new Date());
wfJob.setWorkflowInstance(workflowInstance);
Status slaStatus = Status.SUCCEEDED;
switch (wfJob.getStatus()) {
case SUCCEEDED:
slaStatus = Status.SUCCEEDED;
break;
case KILLED:
slaStatus = Status.KILLED;
break;
case FAILED:
slaStatus = Status.FAILED;
break;
default: // TODO SUSPENDED
break;
}
SLADbXOperations.writeStausEvent(
wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB);
queue(new NotificationXCommand(wfJob));
if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) {
InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation());
}
} else {
for (WorkflowActionBean newAction :
WorkflowStoreService.getStartedActions(workflowInstance)) {
String skipVar =
workflowInstance.getVar(
newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
boolean skipNewAction = false;
if (skipVar != null) {
skipNewAction = skipVar.equals("true");
}
try {
if (skipNewAction) {
WorkflowActionBean oldAction;
oldAction = jpaService.execute(new WorkflowActionGetJPAExecutor(newAction.getId()));
oldAction.setPending();
jpaService.execute(new WorkflowActionUpdateJPAExecutor(oldAction));
queue(new SignalXCommand(jobId, oldAction.getId()));
} else {
newAction.setPending();
String actionSlaXml =
getActionSLAXml(
newAction.getName(),
workflowInstance.getApp().getDefinition(),
wfJob.getConf());
newAction.setSlaXml(actionSlaXml);
jpaService.execute(new WorkflowActionInsertJPAExecutor(newAction));
LOG.debug(
"SignalXCommand: Name: "
+ newAction.getName()
+ ", Id: "
+ newAction.getId()
+ ", Authcode:"
+ newAction.getCred());
queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
}
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
}
}
try {
jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob));
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
XLog.getLog(getClass())
.debug(
"Updated the workflow status to "
+ wfJob.getId()
+ " status ="
+ wfJob.getStatusStr());
if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
&& wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) {
// update coordinator action
new CoordActionUpdateXCommand(wfJob).call();
new WfEndXCommand(wfJob).call(); // To delete the WF temp dir
}
LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
return null;
}
public class SignalXCommand extends WorkflowXCommand<Void> {
protected static final String INSTR_SUCCEEDED_JOBS_COUNTER_NAME = "succeeded";
private final XLog LOG = XLog.getLog(getClass());
private JPAService jpaService = null;
private String jobId;
private String actionId;
private WorkflowJobBean wfJob;
private WorkflowActionBean wfAction;
public SignalXCommand(String name, int priority, String jobId) {
super(name, name, priority);
this.jobId = ParamChecker.notEmpty(jobId, "jobId");
}
public SignalXCommand(String jobId, String actionId) {
this("signal", 1, jobId);
this.actionId = ParamChecker.notEmpty(actionId, "actionId");
}
@Override
protected boolean isLockRequired() {
return true;
}
@Override
protected String getEntityKey() {
return this.jobId;
}
@Override
protected void loadState() throws CommandException {
try {
jpaService = Services.get().get(JPAService.class);
if (jpaService != null) {
this.wfJob = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
LogUtils.setLogInfo(wfJob, logInfo);
if (actionId != null) {
this.wfAction = jpaService.execute(new WorkflowActionGetJPAExecutor(actionId));
LogUtils.setLogInfo(wfAction, logInfo);
}
} else {
throw new CommandException(ErrorCode.E0610);
}
} catch (XException ex) {
throw new CommandException(ex);
}
}
@Override
protected void verifyPrecondition() throws CommandException, PreconditionException {
if ((wfAction == null) || (wfAction.isComplete() && wfAction.isPending())) {
if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
&& wfJob.getStatus() != WorkflowJob.Status.PREP) {
throw new PreconditionException(ErrorCode.E0813, wfJob.getStatusStr());
}
} else {
throw new PreconditionException(
ErrorCode.E0814, actionId, wfAction.getStatusStr(), wfAction.isPending());
}
}
@Override
protected Void execute() throws CommandException {
LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
WorkflowInstance workflowInstance = wfJob.getWorkflowInstance();
workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob);
boolean completed = false;
boolean skipAction = false;
if (wfAction == null) {
if (wfJob.getStatus() == WorkflowJob.Status.PREP) {
try {
completed = workflowInstance.start();
} catch (WorkflowException e) {
throw new CommandException(e);
}
wfJob.setStatus(WorkflowJob.Status.RUNNING);
wfJob.setStartTime(new Date());
wfJob.setWorkflowInstance(workflowInstance);
// 1. Add SLA status event for WF-JOB with status STARTED
// 2. Add SLA registration events for all WF_ACTIONS
SLADbXOperations.writeStausEvent(
wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB);
writeSLARegistrationForAllActions(
workflowInstance.getApp().getDefinition(),
wfJob.getUser(),
wfJob.getGroup(),
wfJob.getConf());
queue(new NotificationXCommand(wfJob));
} else {
throw new CommandException(ErrorCode.E0801, wfJob.getId());
}
} else {
String skipVar =
workflowInstance.getVar(
wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
if (skipVar != null) {
skipAction = skipVar.equals("true");
}
try {
completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue());
} catch (WorkflowException e) {
throw new CommandException(e);
}
wfJob.setWorkflowInstance(workflowInstance);
wfAction.resetPending();
if (!skipAction) {
wfAction.setTransition(workflowInstance.getTransition(wfAction.getName()));
}
try {
jpaService.execute(new WorkflowActionUpdateJPAExecutor(wfAction));
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
}
if (completed) {
try {
for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) {
WorkflowActionBean actionToKill;
actionToKill = jpaService.execute(new WorkflowActionGetJPAExecutor(actionToKillId));
actionToKill.setPending();
actionToKill.setStatus(WorkflowActionBean.Status.KILLED);
jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToKill));
queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType()));
}
for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) {
WorkflowActionBean actionToFail =
jpaService.execute(new WorkflowActionGetJPAExecutor(actionToFailId));
actionToFail.resetPending();
actionToFail.setStatus(WorkflowActionBean.Status.FAILED);
SLADbXOperations.writeStausEvent(
wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION);
jpaService.execute(new WorkflowActionUpdateJPAExecutor(actionToFail));
}
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
wfJob.setEndTime(new Date());
wfJob.setWorkflowInstance(workflowInstance);
Status slaStatus = Status.SUCCEEDED;
switch (wfJob.getStatus()) {
case SUCCEEDED:
slaStatus = Status.SUCCEEDED;
break;
case KILLED:
slaStatus = Status.KILLED;
break;
case FAILED:
slaStatus = Status.FAILED;
break;
default: // TODO SUSPENDED
break;
}
SLADbXOperations.writeStausEvent(
wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB);
queue(new NotificationXCommand(wfJob));
if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) {
InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation());
}
} else {
for (WorkflowActionBean newAction :
WorkflowStoreService.getStartedActions(workflowInstance)) {
String skipVar =
workflowInstance.getVar(
newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunCommand.TO_SKIP);
boolean skipNewAction = false;
if (skipVar != null) {
skipNewAction = skipVar.equals("true");
}
try {
if (skipNewAction) {
WorkflowActionBean oldAction;
oldAction = jpaService.execute(new WorkflowActionGetJPAExecutor(newAction.getId()));
oldAction.setPending();
jpaService.execute(new WorkflowActionUpdateJPAExecutor(oldAction));
queue(new SignalXCommand(jobId, oldAction.getId()));
} else {
newAction.setPending();
String actionSlaXml =
getActionSLAXml(
newAction.getName(),
workflowInstance.getApp().getDefinition(),
wfJob.getConf());
newAction.setSlaXml(actionSlaXml);
jpaService.execute(new WorkflowActionInsertJPAExecutor(newAction));
LOG.debug(
"SignalXCommand: Name: "
+ newAction.getName()
+ ", Id: "
+ newAction.getId()
+ ", Authcode:"
+ newAction.getCred());
queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
}
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
}
}
try {
jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob));
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
XLog.getLog(getClass())
.debug(
"Updated the workflow status to "
+ wfJob.getId()
+ " status ="
+ wfJob.getStatusStr());
if (wfJob.getStatus() != WorkflowJob.Status.RUNNING
&& wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) {
// update coordinator action
new CoordActionUpdateXCommand(wfJob).call();
new WfEndXCommand(wfJob).call(); // To delete the WF temp dir
}
LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
return null;
}
public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
for (Map.Entry<String, String> entry : conf) {
eval.setVariable(entry.getKey(), entry.getValue());
}
return eval;
}
@SuppressWarnings("unchecked")
private String getActionSLAXml(String actionName, String wfXml, String wfConf)
throws CommandException {
String slaXml = null;
try {
Element eWfJob = XmlUtils.parseXml(wfXml);
for (Element action : (List<Element>) eWfJob.getChildren("action", eWfJob.getNamespace())) {
if (action.getAttributeValue("name").equals(actionName) == false) {
continue;
}
Element eSla =
action.getChild("info", Namespace.getNamespace(SchemaService.SLA_NAME_SPACE_URI));
if (eSla != null) {
slaXml = XmlUtils.prettyPrint(eSla).toString();
break;
}
}
} catch (Exception e) {
throw new CommandException(ErrorCode.E1004, e.getMessage(), e);
}
return slaXml;
}
private String resolveSla(Element eSla, Configuration conf) throws CommandException {
String slaXml = null;
try {
ELEvaluator evalSla = SubmitCommand.createELEvaluatorForGroup(conf, "wf-sla-submit");
slaXml = SubmitCommand.resolveSla(eSla, evalSla);
} catch (Exception e) {
throw new CommandException(ErrorCode.E1004, e.getMessage(), e);
}
return slaXml;
}
@SuppressWarnings("unchecked")
private void writeSLARegistrationForAllActions(
String wfXml, String user, String group, String strConf) throws CommandException {
try {
Element eWfJob = XmlUtils.parseXml(wfXml);
Configuration conf = new XConfiguration(new StringReader(strConf));
for (Element action : (List<Element>) eWfJob.getChildren("action", eWfJob.getNamespace())) {
Element eSla =
action.getChild("info", Namespace.getNamespace(SchemaService.SLA_NAME_SPACE_URI));
if (eSla != null) {
String slaXml = resolveSla(eSla, conf);
eSla = XmlUtils.parseXml(slaXml);
String actionId =
Services.get()
.get(UUIDService.class)
.generateChildId(jobId, action.getAttributeValue("name") + "");
SLADbXOperations.writeSlaRegistrationEvent(
eSla, actionId, SlaAppType.WORKFLOW_ACTION, user, group);
}
}
} catch (Exception e) {
throw new CommandException(ErrorCode.E1007, "workflow:Actions " + jobId, e);
}
}
}
protected Void call(WorkflowStore store) throws StoreException, CommandException {
WorkflowJobBean workflow = store.getWorkflow(jobId, false);
setLogInfo(workflow);
WorkflowActionBean action = store.getAction(id, false);
setLogInfo(action);
if (action.isPending()
&& (action.getStatus() == WorkflowActionBean.Status.DONE
|| action.getStatus() == WorkflowActionBean.Status.END_RETRY
|| action.getStatus() == WorkflowActionBean.Status.END_MANUAL)) {
if (workflow.getStatus() == WorkflowJob.Status.RUNNING) {
ActionExecutor executor =
Services.get().get(ActionService.class).getExecutor(action.getType());
Configuration conf = workflow.getWorkflowInstance().getConf();
int maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries());
long retryInterval =
conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval());
executor.setMaxRetries(maxRetries);
executor.setRetryInterval(retryInterval);
if (executor != null) {
boolean isRetry = false;
if (action.getStatus() == WorkflowActionBean.Status.END_RETRY
|| action.getStatus() == WorkflowActionBean.Status.END_MANUAL) {
isRetry = true;
}
ActionExecutorContext context =
new ActionCommand.ActionExecutorContext(workflow, action, isRetry);
try {
XLog.getLog(getClass())
.debug(
"End, name [{0}] type [{1}] status[{2}] external status [{3}] signal value [{4}]",
action.getName(),
action.getType(),
action.getStatus(),
action.getExternalStatus(),
action.getSignalValue());
WorkflowInstance wfInstance = workflow.getWorkflowInstance();
DagELFunctions.setActionInfo(wfInstance, action);
workflow.setWorkflowInstance(wfInstance);
incrActionCounter(action.getType(), 1);
Instrumentation.Cron cron = new Instrumentation.Cron();
cron.start();
executor.end(context, action);
cron.stop();
addActionCron(action.getType(), cron);
if (!context.isEnded()) {
XLog.getLog(getClass())
.warn(
XLog.OPS,
"Action Ended, ActionExecutor [{0}] must call setEndData()",
executor.getType());
action.setErrorInfo(
END_DATA_MISSING, "Execution Ended, but End Data Missing from Action");
failJob(context);
store.updateAction(action);
store.updateWorkflow(workflow);
return null;
}
action.setRetries(0);
action.setEndTime(new Date());
store.updateAction(action);
store.updateWorkflow(workflow);
Status slaStatus = null;
switch (action.getStatus()) {
case OK:
slaStatus = Status.SUCCEEDED;
break;
case KILLED:
slaStatus = Status.KILLED;
break;
case FAILED:
slaStatus = Status.FAILED;
break;
case ERROR:
XLog.getLog(getClass()).info("ERROR is considered as FAILED for SLA");
slaStatus = Status.KILLED;
break;
default: // TODO: What will happen for other Action
// status
slaStatus = Status.FAILED;
break;
}
SLADbOperations.writeStausEvent(
action.getSlaXml(), action.getId(), store, slaStatus, SlaAppType.WORKFLOW_ACTION);
queueCallable(new NotificationCommand(workflow, action));
XLog.getLog(getClass())
.debug(
"Queuing commands for action "
+ id
+ " status "
+ action.getStatus()
+ ", Set pending="
+ action.getPending());
queueCallable(new SignalCommand(workflow.getId(), id));
} catch (ActionExecutorException ex) {
XLog.getLog(getClass())
.warn(
"Error ending action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]",
action.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage());
action.setErrorInfo(ex.getErrorCode(), ex.getMessage());
action.setEndTime(null);
switch (ex.getErrorType()) {
case TRANSIENT:
if (!handleTransient(context, executor, WorkflowAction.Status.END_RETRY)) {
handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL);
action.setPendingAge(new Date());
action.setRetries(0);
}
action.setEndTime(null);
break;
case NON_TRANSIENT:
handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL);
action.setEndTime(null);
break;
case ERROR:
handleError(context, executor, COULD_NOT_END, false, WorkflowAction.Status.ERROR);
queueCallable(new SignalCommand(workflow.getId(), id));
break;
case FAILED:
failJob(context);
break;
}
store.updateAction(action);
store.updateWorkflow(workflow);
}
} else {
throw new CommandException(ErrorCode.E0802, action.getType());
}
} else {
XLog.getLog(getClass())
.warn(
"Job state is not {0}. Skipping ActionEnd Execution",
WorkflowJob.Status.RUNNING.toString());
}
} else {
XLog.getLog(getClass())
.debug(
"Action pending={0}, status={1}. Skipping ActionEnd Execution",
action.getPending(), action.getStatusStr());
}
return null;
}
protected LiteWorkflowInstance() {
log = XLog.getLog(getClass());
}
public static SLARegistrationBean createSlaRegistrationEvent(
Element eSla,
String jobId,
String parentId,
AppType appType,
String user,
String appName,
XLog log,
boolean rerun)
throws CommandException {
if (eSla == null || !SLAService.isEnabled()) {
log.debug("Not registering SLA for job [{0}]. Sla-Xml null OR SLAService not enabled", jobId);
return null;
}
SLARegistrationBean sla = new SLARegistrationBean();
// Setting nominal time
String strNominalTime = getTagElement(eSla, NOMINAL_TIME);
if (strNominalTime == null || strNominalTime.length() == 0) {
throw new CommandException(ErrorCode.E1101, NOMINAL_TIME);
}
Date nominalTime;
try {
nominalTime = DateUtils.parseDateOozieTZ(strNominalTime);
sla.setNominalTime(nominalTime);
} catch (ParseException pex) {
throw new CommandException(ErrorCode.E0302, strNominalTime, pex);
}
// Setting expected start time
String strExpectedStart = getTagElement(eSla, SHOULD_START);
if (strExpectedStart != null) {
float expectedStart = Float.parseFloat(strExpectedStart);
if (expectedStart < 0) {
throw new CommandException(
ErrorCode.E0302, strExpectedStart, "for SLA Expected start time");
} else {
Date expectedStartTime =
new Date(nominalTime.getTime() + (long) (expectedStart * 60 * 1000));
sla.setExpectedStart(expectedStartTime);
}
}
// Setting expected end time
String strExpectedEnd = getTagElement(eSla, SHOULD_END);
if (strExpectedEnd == null || strExpectedEnd.length() == 0) {
throw new CommandException(ErrorCode.E1101, SHOULD_END);
}
float expectedEnd = Float.parseFloat(strExpectedEnd);
if (expectedEnd < 0) {
throw new CommandException(ErrorCode.E0302, strExpectedEnd, "for SLA Expected end time");
} else {
Date expectedEndTime = new Date(nominalTime.getTime() + (long) (expectedEnd * 60 * 1000));
sla.setExpectedEnd(expectedEndTime);
}
// Setting expected duration in milliseconds
String expectedDurationStr = getTagElement(eSla, MAX_DURATION);
if (expectedDurationStr != null && expectedDurationStr.length() > 0) {
float expectedDuration = Float.parseFloat(expectedDurationStr);
if (expectedDuration > 0) {
sla.setExpectedDuration((long) (expectedDuration * 60 * 1000));
}
} else if (sla.getExpectedStart() != null) {
sla.setExpectedDuration(sla.getExpectedEnd().getTime() - sla.getExpectedStart().getTime());
}
// Parse desired alert-types i.e. start-miss, end-miss, start-met etc..
String alertEvents = getTagElement(eSla, ALERT_EVENTS);
if (alertEvents != null) {
String events[] = alertEvents.split(",");
StringBuilder alertsStr = new StringBuilder();
for (int i = 0; i < events.length; i++) {
String event = events[i].trim().toUpperCase();
try {
EventStatus.valueOf(event);
} catch (IllegalArgumentException iae) {
XLog.getLog(SLAService.class)
.warn(
"Invalid value: ["
+ event
+ "]"
+ " for SLA Alert-event. Should be one of "
+ EventStatus.values()
+ ". Setting it to default ["
+ EventStatus.END_MISS.name()
+ "]");
event = EventStatus.END_MISS.name();
}
alertsStr.append(event).append(",");
}
sla.setAlertEvents(alertsStr.toString().substring(0, alertsStr.lastIndexOf(",")));
}
// Other sla config
sla.setNotificationMsg(getTagElement(eSla, "notification-msg"));
sla.setAlertContact(getTagElement(eSla, "alert-contact"));
sla.setUpstreamApps(getTagElement(eSla, "upstream-apps"));
// Oozie defined
sla.setId(jobId);
sla.setAppType(appType);
sla.setAppName(appName);
sla.setUser(user);
sla.setParentId(parentId);
SLAService slaService = Services.get().get(SLAService.class);
try {
if (!rerun) {
slaService.addRegistrationEvent(sla);
} else {
slaService.updateRegistrationEvent(sla);
}
} catch (ServiceException e) {
throw new CommandException(ErrorCode.E1007, " id " + jobId, e.getMessage(), e);
}
log.debug(
"Job [{0}] reg for SLA. Size of Sla Xml = [{1}]",
jobId, XmlUtils.prettyPrint(eSla).toString().length());
return sla;
}
/**
* Service that provides distributed job id sequence via ZooKeeper. Requires that a ZooKeeper
* ensemble is available. The sequence path will be located under a ZNode named "job_id_sequence"
* under the namespace (see {@link ZKUtils}). The sequence will be reset to 0, once max is reached.
*/
public class ZKUUIDService extends UUIDService {
public static final String CONF_PREFIX = Service.CONF_PREFIX + "ZKUUIDService.";
public static final String CONF_SEQUENCE_MAX = CONF_PREFIX + "jobid.sequence.max";
public static final String ZK_SEQUENCE_PATH = "job_id_sequence";
public static final long RESET_VALUE = 0l;
public static final int RETRY_COUNT = 3;
private static final XLog LOG = XLog.getLog(ZKUUIDService.class);
private ZKUtils zk;
private static Long maxSequence = 9999990l;
DistributedAtomicLong atomicIdGenerator;
@Override
public void init(Services services) throws ServiceException {
super.init(services);
try {
zk = ZKUtils.register(this);
atomicIdGenerator =
new DistributedAtomicLong(zk.getClient(), ZK_SEQUENCE_PATH, ZKUtils.getRetryPloicy());
} catch (Exception ex) {
throw new ServiceException(ErrorCode.E1700, ex.getMessage(), ex);
}
}
/**
* Gets the unique id.
*
* @return the id
* @throws Exception the exception
*/
public long getID() {
return getZKId(0);
}
@SuppressWarnings("finally")
private long getZKId(int retryCount) {
if (atomicIdGenerator == null) {
throw new RuntimeException("Sequence generator can't be null. Path : " + ZK_SEQUENCE_PATH);
}
AtomicValue<Long> value = null;
try {
value = atomicIdGenerator.increment();
} catch (Exception e) {
throw new RuntimeException("Exception incrementing UID for session ", e);
} finally {
if (value != null && value.succeeded()) {
if (value.preValue() >= maxSequence) {
if (retryCount >= RETRY_COUNT) {
throw new RuntimeException("Can't reset sequence. Tried " + retryCount + " times");
}
resetSequence();
return getZKId(retryCount + 1);
}
return value.preValue();
} else {
throw new RuntimeException("Exception incrementing UID for session ");
}
}
}
/** Once sequence is reached limit, reset to 0. */
private void resetSequence() {
synchronized (ZKUUIDService.class) {
try {
// Double check if sequence is already reset.
AtomicValue<Long> value = atomicIdGenerator.get();
if (value.succeeded()) {
if (value.postValue() < maxSequence) {
return;
}
} else {
throw new RuntimeException("Can't reset sequence");
}
// Acquire ZK lock, so that other host doesn't reset sequence.
LockToken lock =
Services.get()
.get(MemoryLocksService.class)
.getWriteLock(ZKUUIDService.class.getName(), lockTimeout);
try {
if (lock == null) {
LOG.info("Lock is held by other system, returning");
return;
} else {
value = atomicIdGenerator.get();
if (value.succeeded()) {
if (value.postValue() < maxSequence) {
return;
}
} else {
throw new RuntimeException("Can't reset sequence");
}
atomicIdGenerator.forceSet(RESET_VALUE);
resetStartTime();
}
} finally {
if (lock != null) {
lock.release();
}
}
} catch (Exception e) {
throw new RuntimeException("Can't reset sequence", e);
}
}
}
@Override
public void destroy() {
if (zk != null) {
zk.unregister(this);
}
zk = null;
super.destroy();
}
@VisibleForTesting
public void setMaxSequence(long sequence) {
maxSequence = sequence;
}
}
