/**
* Run recipe with different frequencies. Submission should go through. Check frequency of the
* launched oozie job
*/
@Test(dataProvider = "frequencyGenerator")
public void differentRecipeFrequenciesTest(String frequency) throws Exception {
setUp(RecipeExecLocation.SourceCluster);
LOGGER.info("Testing with frequency: " + frequency);
String tblName = "myTable";
recipeMerlin
.withSourceDb(DB_NAME)
.withSourceTable(tblName)
.withFrequency(new Frequency(frequency));
runSql(connection, "create table " + tblName + "(comment string)");
final List<String> command = recipeMerlin.getSubmissionCommand();
Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
LOGGER.info("Submission went through.");
InstanceUtil.waitTillInstanceReachState(
clusterOC, recipeMerlin.getName(), 1, CoordinatorAction.Status.RUNNING, EntityType.PROCESS);
String filter = "name=FALCON_PROCESS_" + recipeMerlin.getName();
List<BundleJob> bundleJobs = OozieUtil.getBundles(clusterOC, filter, 0, 10);
List<String> bundleIds = OozieUtil.getBundleIds(bundleJobs);
String bundleId = OozieUtil.getMaxId(bundleIds);
List<CoordinatorJob> coords = clusterOC.getBundleJobInfo(bundleId).getCoordinators();
List<String> cIds = new ArrayList<String>();
for (CoordinatorJob coord : coords) {
cIds.add(coord.getId());
}
String coordId = OozieUtil.getMinId(cIds);
CoordinatorJob job = clusterOC.getCoordJobInfo(coordId);
CoordinatorJob.Timeunit timeUnit = job.getTimeUnit();
String freq = job.getFrequency();
LOGGER.info("Frequency of running job: " + timeUnit + " " + freq);
Assert.assertTrue(
frequency.contains(timeUnit.name().toLowerCase().replace("_", ""))
&& frequency.contains(freq),
"Running job has different frequency.");
}
/**
* Resolve basic entities using job Configuration.
*
* @param conf :Job configuration
* @param appXml : Original job XML
* @param coordJob : Coordinator job bean to be populated.
* @return Resolved job XML element.
* @throws CoordinatorJobException thrown if failed to resolve basic entities
* @throws Exception thrown if failed to resolve basic entities
*/
@SuppressWarnings("unchecked")
protected Element resolveInitial(Configuration conf, String appXml, CoordinatorJobBean coordJob)
throws CoordinatorJobException, Exception {
Element eAppXml = XmlUtils.parseXml(appXml);
// job's main attributes
// frequency
String val = resolveAttribute("frequency", eAppXml, evalFreq);
int ival = ParamChecker.checkInteger(val, "frequency");
ParamChecker.checkGTZero(ival, "frequency");
coordJob.setFrequency(ival);
TimeUnit tmp =
(evalFreq.getVariable("timeunit") == null)
? TimeUnit.MINUTE
: ((TimeUnit) evalFreq.getVariable("timeunit"));
addAnAttribute("freq_timeunit", eAppXml, tmp.toString());
// TimeUnit
coordJob.setTimeUnit(CoordinatorJob.Timeunit.valueOf(tmp.toString()));
// End Of Duration
tmp =
evalFreq.getVariable("endOfDuration") == null
? TimeUnit.NONE
: ((TimeUnit) evalFreq.getVariable("endOfDuration"));
addAnAttribute("end_of_duration", eAppXml, tmp.toString());
// coordJob.setEndOfDuration(tmp) // TODO: Add new attribute in Job bean
// Application name
if (this.coordName == null) {
val = resolveAttribute("name", eAppXml, evalNofuncs);
coordJob.setAppName(val);
} else {
// this coord job is created from bundle
coordJob.setAppName(this.coordName);
}
// start time
val = resolveAttribute("start", eAppXml, evalNofuncs);
ParamChecker.checkUTC(val, "start");
coordJob.setStartTime(DateUtils.parseDateUTC(val));
// end time
val = resolveAttribute("end", eAppXml, evalNofuncs);
ParamChecker.checkUTC(val, "end");
coordJob.setEndTime(DateUtils.parseDateUTC(val));
// Time zone
val = resolveAttribute("timezone", eAppXml, evalNofuncs);
ParamChecker.checkTimeZone(val, "timezone");
coordJob.setTimeZone(val);
// controls
val =
resolveTagContents(
"timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
if (val == "") {
val = Services.get().getConf().get(CONF_DEFAULT_TIMEOUT_NORMAL);
}
ival = ParamChecker.checkInteger(val, "timeout");
if (ival < 0 || ival > Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600)) {
ival = Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600);
}
coordJob.setTimeout(ival);
val =
resolveTagContents(
"concurrency", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
if (val == null || val.isEmpty()) {
val = Services.get().getConf().get(CONF_DEFAULT_CONCURRENCY, "-1");
}
ival = ParamChecker.checkInteger(val, "concurrency");
coordJob.setConcurrency(ival);
val =
resolveTagContents(
"throttle", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
if (val == null || val.isEmpty()) {
int defaultThrottle = Services.get().getConf().getInt(CONF_DEFAULT_THROTTLE, 12);
ival = defaultThrottle;
} else {
ival = ParamChecker.checkInteger(val, "throttle");
}
int maxQueue = Services.get().getConf().getInt(CONF_QUEUE_SIZE, 10000);
float factor = Services.get().getConf().getFloat(CONF_MAT_THROTTLING_FACTOR, 0.10f);
int maxThrottle = (int) (maxQueue * factor);
if (ival > maxThrottle || ival < 1) {
ival = maxThrottle;
}
LOG.debug("max throttle " + ival);
coordJob.setMatThrottling(ival);
val =
resolveTagContents(
"execution", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
if (val == "") {
val = Execution.FIFO.toString();
}
coordJob.setExecution(Execution.valueOf(val));
String[] acceptedVals = {
Execution.LIFO.toString(), Execution.FIFO.toString(), Execution.LAST_ONLY.toString()
};
ParamChecker.isMember(val, acceptedVals, "execution");
// datasets
resolveTagContents(
"include", eAppXml.getChild("datasets", eAppXml.getNamespace()), evalNofuncs);
// for each data set
resolveDataSets(eAppXml);
HashMap<String, String> dataNameList = new HashMap<String, String>();
resolveIOEvents(eAppXml, dataNameList);
resolveTagContents(
"app-path",
eAppXml
.getChild("action", eAppXml.getNamespace())
.getChild("workflow", eAppXml.getNamespace()),
evalNofuncs);
// TODO: If action or workflow tag is missing, NullPointerException will
// occur
Element configElem =
eAppXml
.getChild("action", eAppXml.getNamespace())
.getChild("workflow", eAppXml.getNamespace())
.getChild("configuration", eAppXml.getNamespace());
evalData =
CoordELEvaluator.createELEvaluatorForDataEcho(conf, "coord-job-submit-data", dataNameList);
if (configElem != null) {
for (Element propElem :
(List<Element>) configElem.getChildren("property", configElem.getNamespace())) {
resolveTagContents("name", propElem, evalData);
// Want to check the data-integrity but don't want to modify the
// XML
// for properties only
Element tmpProp = (Element) propElem.clone();
resolveTagContents("value", tmpProp, evalData);
}
}
resolveSLA(eAppXml, coordJob);
return eAppXml;
}