private void timesClause(StringBuilder sb, int offset, Map<String, Timestamp> eachTime)
throws ParseException {
Timestamp ts = null;
List<String> times = bulkFilter.get(BulkResponseImpl.BULK_FILTER_START_CREATED_EPOCH);
if (times != null) {
ts = new Timestamp(DateUtils.parseDateUTC(times.get(0)).getTime());
sb.insert(offset - 1, " AND a.createdTimestamp >= :startCreated");
eachTime.put("startCreated", ts);
}
times = bulkFilter.get(BulkResponseImpl.BULK_FILTER_END_CREATED_EPOCH);
if (times != null) {
ts = new Timestamp(DateUtils.parseDateUTC(times.get(0)).getTime());
sb.insert(offset - 1, " AND a.createdTimestamp <= :endCreated");
eachTime.put("endCreated", ts);
}
times = bulkFilter.get(BulkResponseImpl.BULK_FILTER_START_NOMINAL_EPOCH);
if (times != null) {
ts = new Timestamp(DateUtils.parseDateUTC(times.get(0)).getTime());
sb.insert(offset - 1, " AND a.nominalTimestamp >= :startNominal");
eachTime.put("startNominal", ts);
}
times = bulkFilter.get(BulkResponseImpl.BULK_FILTER_END_NOMINAL_EPOCH);
if (times != null) {
ts = new Timestamp(DateUtils.parseDateUTC(times.get(0)).getTime());
sb.insert(offset - 1, " AND a.nominalTimestamp <= :endNominal");
eachTime.put("endNominal", ts);
}
}
public void testAbsoluteRange() throws Exception {
init("coord-action-create");
ds = new SyncCoordDataset();
ds.setFrequency(7);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"));
ds.setTimeUnit(TimeUnit.DAY);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setName("test");
ds.setUriTemplate("hdfs:///tmp/workflows/${YEAR}/${MONTH}/${DAY};region=us");
ds.setType("SYNC");
ds.setDoneFlag("");
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
String expr = "${coord:absoluteRange(\"2009-08-20T01:00Z\",\"0\")}";
assertEquals(CoordELFunctions.evalAndWrap(eval, expr), "2009-08-20T01:00Z");
expr = "${coord:absoluteRange(\"2009-08-20T01:00Z\",\"1\")}";
assertEquals(CoordELFunctions.evalAndWrap(eval, expr), "2009-08-27T01:00Z#2009-08-20T01:00Z");
try {
expr = "${coord:absoluteRange(\"2009-08-20T01:00Z\",\"-1\")}";
CoordELFunctions.evalAndWrap(eval, expr);
fail("start-instance is greater than the end-instance and there was no exception");
} catch (Exception e) {
assertTrue(
e.getCause()
.getMessage()
.contains("start-instance should be equal or earlier than the end-instance"));
}
}
/**
* Returns the a date string while given a base date in 'strBaseDate', offset and unit (e.g. DAY,
* MONTH, HOUR, MINUTE, MONTH).
*
* @param strBaseDate -- base date
* @param offset -- any number
* @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH
* @return date string
* @throws Exception
*/
public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit)
throws Exception {
Calendar baseCalDate = DateUtils.getCalendar(strBaseDate);
StringBuilder buffer = new StringBuilder();
baseCalDate.add(TimeUnit.valueOf(unit).getCalendarUnit(), offset);
buffer.append(DateUtils.formatDateUTC(baseCalDate));
return buffer.toString();
}
public void testActionMater() throws Exception {
Date startTime = DateUtils.parseDateUTC("2009-03-06T010:00Z");
Date endTime = DateUtils.parseDateUTC("2009-03-11T10:00Z");
CoordinatorJobBean job =
addRecordToCoordJobTable(CoordinatorJob.Status.PREMATER, startTime, endTime);
new CoordActionMaterializeXCommand(job.getId(), startTime, endTime).call();
CoordinatorActionBean action = checkCoordAction(job.getId() + "@1");
}
public void testActionMaterWithPauseTime3() throws Exception {
Date startTime = DateUtils.parseDateUTC("2009-03-06T10:00Z");
Date endTime = DateUtils.parseDateUTC("2009-03-06T10:14Z");
Date pauseTime = DateUtils.parseDateUTC("2009-03-06T09:58Z");
CoordinatorJobBean job =
addRecordToCoordJobTable(CoordinatorJob.Status.PREMATER, startTime, endTime, pauseTime);
new CoordActionMaterializeXCommand(job.getId(), startTime, endTime).call();
checkCoordActions(job.getId(), 0, CoordinatorJob.Status.RUNNING);
}
private Element createActionElement(String actionXml) throws JDOMException, ParseException {
Element eAction = XmlUtils.parseXml(actionXml);
eAction.removeAttribute("start");
eAction.removeAttribute("end");
eAction.setAttribute("instance-number", Integer.toString(1));
eAction.setAttribute(
"action-nominal-time",
DateUtils.formatDateOozieTZ(DateUtils.parseDateOozieTZ("2009-09-08T01:00Z")));
eAction.setAttribute("action-actual-time", DateUtils.formatDateOozieTZ(new Date()));
return eAction;
}
public void testOnSLADurationMetEvent() throws Exception {
JMSSLAEventListener slaListener = new JMSSLAEventListener();
slaListener.init(conf);
SLACalcStatus durationMet = new SLACalcStatus(new SLARegistrationBean());
SLARegistrationBean durationMetBean = durationMet.getSLARegistrationBean();
Date expectedStartDate = DateUtils.parseDateUTC("2013-01-01T00:00Z");
Date actualStartDate = DateUtils.parseDateUTC("2013-01-01T01:00Z");
Date expectedEndDate = DateUtils.parseDateUTC("2013-01-01T12:00Z");
Date actualEndDate = DateUtils.parseDateUTC("2013-01-01T14:00Z");
long expectedDuration = (expectedEndDate.getTime() - actualStartDate.getTime()) / (1000 * 60);
durationMet.setId("0000000-000000000000001-oozie-wrkf-C@1");
durationMetBean.setParentId("0000000-000000000000001-oozie-wrkf-C");
durationMetBean.setAppName("Test-SLA-Duration-Met");
durationMet.setEventStatus(EventStatus.DURATION_MET);
durationMet.setSLAStatus(SLAStatus.MET);
durationMetBean.setAppType(AppType.COORDINATOR_ACTION);
durationMetBean.setUser("dummyuser");
durationMetBean.setNotificationMsg("notification of duration met");
durationMetBean.setExpectedStart(expectedStartDate);
durationMet.setActualStart(actualStartDate);
durationMetBean.setExpectedEnd(expectedEndDate);
durationMet.setActualEnd(actualEndDate);
durationMetBean.setExpectedDuration(expectedDuration);
long actualDuration = actualEndDate.getTime() - actualStartDate.getTime();
durationMet.setActualDuration(actualDuration);
ConnectionContext jmsContext = getConnectionContext();
Session session = jmsContext.createSession(Session.AUTO_ACKNOWLEDGE);
MessageConsumer consumer =
jmsContext.createConsumer(session, slaListener.getTopic(durationMet));
slaListener.onDurationMet(durationMet);
TextMessage message = (TextMessage) consumer.receive(5000);
SLAMessage durationMissMsg = (SLAMessage) JMSMessagingUtils.getEventMessage(message);
// check msg header
assertEquals(EventStatus.DURATION_MET, durationMissMsg.getEventStatus());
assertEquals(SLAStatus.MET, durationMissMsg.getSLAStatus());
assertEquals(AppType.COORDINATOR_ACTION, durationMissMsg.getAppType());
assertEquals(MessageType.SLA, durationMissMsg.getMessageType());
assertEquals("Test-SLA-Duration-Met", durationMissMsg.getAppName());
assertEquals("dummyuser", durationMissMsg.getUser());
// check msg body
assertEquals("0000000-000000000000001-oozie-wrkf-C@1", durationMissMsg.getId());
assertEquals("0000000-000000000000001-oozie-wrkf-C", durationMissMsg.getParentId());
assertEquals(expectedStartDate, durationMissMsg.getExpectedStartTime());
assertEquals(actualStartDate, durationMissMsg.getActualStartTime());
assertEquals(expectedEndDate, durationMissMsg.getExpectedEndTime());
assertEquals(actualEndDate, durationMissMsg.getActualEndTime());
assertEquals(expectedDuration, durationMissMsg.getExpectedDuration());
assertEquals(actualDuration, durationMissMsg.getActualDuration());
assertEquals("notification of duration met", durationMissMsg.getNotificationMessage());
}
public void testTimeout() throws Exception {
Date startTime = DateUtils.parseDateUTC("2009-03-06T10:00Z");
Date endTime = DateUtils.parseDateUTC("2009-03-06T10:14Z");
Date pauseTime = null;
CoordinatorJobBean job =
addRecordToCoordJobTable(
CoordinatorJob.Status.PREMATER, startTime, endTime, pauseTime, 300);
new CoordActionMaterializeXCommand(job.getId(), startTime, endTime).call();
JPAService jpaService = Services.get().get(JPAService.class);
List<CoordinatorActionBean> actions =
jpaService.execute(new CoordJobGetActionsJPAExecutor(job.getId()));
checkCoordActionsTimeout(job.getId() + "@1", 300);
}
public void testActionMater() throws Exception {
// NOTE: If this test runs multiple times with mysql DB, all the tests
// would fail except the first one.
// To make it work in mysql, you need to remove the records.
// It is intended to be tested against in-memory DB
String jobId = "0000000-" + new Date().getTime() + "-testActionMater-C";
Date startTime = DateUtils.parseDateUTC("2009-03-06T010:00Z");
Date endTime = DateUtils.parseDateUTC("2009-03-11T10:00Z");
addRecordToJobTable(jobId, startTime, endTime);
new CoordActionMaterializeCommand(jobId, startTime, endTime).call();
checkCoordAction(jobId + "@1");
// Thread.sleep(300000);
}
private BulkResponseImpl getResponseFromObject(BundleJobBean bundleBean, Object arr[]) {
BulkResponseImpl bean = new BulkResponseImpl();
CoordinatorJobBean coordBean = new CoordinatorJobBean();
CoordinatorActionBean actionBean = new CoordinatorActionBean();
if (arr[0] != null) {
actionBean.setId((String) arr[0]);
}
if (arr[1] != null) {
actionBean.setActionNumber((Integer) arr[1]);
}
if (arr[2] != null) {
actionBean.setErrorCode((String) arr[2]);
}
if (arr[3] != null) {
actionBean.setErrorMessage((String) arr[3]);
}
if (arr[4] != null) {
actionBean.setExternalId((String) arr[4]);
}
if (arr[5] != null) {
actionBean.setExternalStatus((String) arr[5]);
}
if (arr[6] != null) {
actionBean.setStatus(CoordinatorAction.Status.valueOf((String) arr[6]));
}
if (arr[7] != null) {
actionBean.setCreatedTime(DateUtils.toDate((Timestamp) arr[7]));
}
if (arr[8] != null) {
actionBean.setNominalTime(DateUtils.toDate((Timestamp) arr[8]));
}
if (arr[9] != null) {
actionBean.setMissingDependencies((String) arr[9]);
}
if (arr[10] != null) {
coordBean.setId((String) arr[10]);
actionBean.setJobId((String) arr[10]);
}
if (arr[11] != null) {
coordBean.setAppName((String) arr[11]);
}
if (arr[12] != null) {
coordBean.setStatus(CoordinatorJob.Status.valueOf((String) arr[12]));
}
bean.setBundle(bundleBean);
bean.setCoordinator(coordBean);
bean.setAction(actionBean);
return bean;
}
/**
* Return nominal time or Action Creation Time.
*
* <p>
*
* @return coordinator action creation or materialization date time
* @throws Exception if unable to format the Date object to String
*/
public static String ph2_coord_nominalTime() throws Exception {
ELEvaluator eval = ELEvaluator.getCurrent();
SyncCoordAction action =
ParamChecker.notNull(
(SyncCoordAction) eval.getVariable(COORD_ACTION), "Coordinator Action");
return DateUtils.formatDateUTC(action.getNominalTime());
}
/**
* Return Action Start time.
*
* <p>
*
* @return coordinator action start time
* @throws Exception if unable to format the Date object to String
*/
public static String ph2_coord_actualTime() throws Exception {
ELEvaluator eval = ELEvaluator.getCurrent();
SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
if (coordAction == null) {
throw new RuntimeException(
"Associated Application instance should be defined with key " + COORD_ACTION);
}
return DateUtils.formatDateUTC(coordAction.getActualTime());
}
public void testTZOffset() throws Exception {
init("coord-action-create");
String expr = "${coord:tzOffset()}";
// eval.setVariable("resolve_tzOffset", "true");
assertEquals("0", CoordELFunctions.evalAndWrap(eval, expr));
appInst.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2012-06-13T00:00Z")); // Summer
CoordELFunctions.configureEvaluator(eval, ds, appInst);
// PDT is UTC - 7
assertEquals("-420", CoordELFunctions.evalAndWrap(eval, expr));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2012-12-13T00:00Z")); // Winter
CoordELFunctions.configureEvaluator(eval, ds, appInst);
// PST is UTC - 8
assertEquals("-480", CoordELFunctions.evalAndWrap(eval, expr));
}
/**
* validate if change value is valid.
*
* @param changeValue change value.
* @throws CommandException thrown if changeValue cannot be parsed properly.
*/
private void validateChangeValue(String changeValue) throws CommandException {
Map<String, String> map = JobUtils.parseChangeValue(changeValue);
if (map.size() > ALLOWED_CHANGE_OPTIONS.size()
|| !(map.containsKey(OozieClient.CHANGE_VALUE_PAUSETIME)
|| map.containsKey(OozieClient.CHANGE_VALUE_ENDTIME))) {
throw new CommandException(
ErrorCode.E1317, changeValue, "can only change pausetime or end time");
}
if (map.containsKey(OozieClient.CHANGE_VALUE_PAUSETIME)) {
isChangePauseTime = true;
} else if (map.containsKey(OozieClient.CHANGE_VALUE_ENDTIME)) {
isChangeEndTime = true;
} else {
throw new CommandException(
ErrorCode.E1317, changeValue, "should change pausetime or endtime");
}
if (isChangePauseTime) {
String value = map.get(OozieClient.CHANGE_VALUE_PAUSETIME);
if (!value.equals("")) {
try {
newPauseTime = DateUtils.parseDateOozieTZ(value);
} catch (Exception ex) {
throw new CommandException(ErrorCode.E1317, value, "is not a valid date");
}
checkPauseTime(newPauseTime);
}
} else if (isChangeEndTime) {
String value = map.get(OozieClient.CHANGE_VALUE_ENDTIME);
if (!value.equals("")) {
try {
newEndTime = DateUtils.parseDateOozieTZ(value);
} catch (Exception ex) {
throw new CommandException(ErrorCode.E1317, value, "is not a valid date");
}
checkEndTime(newEndTime);
}
}
}
private void init(String tag, String uriTemplate) throws Exception {
eval = Services.get().get(ELService.class).createEvaluator(tag);
eval.setVariable(OozieClient.USER_NAME, "test_user");
eval.setVariable(OozieClient.GROUP_NAME, "test_group");
appInst = new SyncCoordAction();
ds = new SyncCoordDataset();
ds.setFrequency(1);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-09-01T23:59Z"));
ds.setTimeUnit(TimeUnit.DAY);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setName("test");
ds.setUriTemplate(uriTemplate);
ds.setType("SYNC");
ds.setDoneFlag("");
appInst.setActualTime(DateUtils.parseDateOozieTZ("2009-09-10T23:59Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-09-09T23:59Z"));
appInst.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
appInst.setActionId("00000-oozie-C@1");
appInst.setName("mycoordinator-app");
CoordELFunctions.configureEvaluator(eval, ds, appInst);
}
@Test
public void testOnSLAStartMissEvent() throws Exception {
JMSSLAEventListener slaListener = new JMSSLAEventListener();
slaListener.init(conf);
SLACalcStatus startMiss = new SLACalcStatus(new SLARegistrationBean());
SLARegistrationBean startMissBean = startMiss.getSLARegistrationBean();
Date startDate = DateUtils.parseDateUTC("2013-01-01T00:00Z");
startMiss.setId("0000000-000000000000001-oozie-wrkf-C@1");
startMissBean.setParentId("0000000-000000000000001-oozie-wrkf-C");
startMissBean.setAppName("Test-SLA-Start-Miss");
startMissBean.setUser("dummyuser");
startMissBean.setExpectedStart(startDate);
startMissBean.setNotificationMsg("notification of start miss");
startMissBean.setJobData("random job data");
startMiss.setEventStatus(EventStatus.START_MISS);
startMiss.setSLAStatus(SLAStatus.NOT_STARTED);
startMissBean.setAppType(AppType.COORDINATOR_ACTION);
startMiss.setActualStart(DateUtils.parseDateUTC("2013-01-01T01:00Z"));
ConnectionContext jmsContext = getConnectionContext();
Session session = jmsContext.createSession(Session.AUTO_ACKNOWLEDGE);
MessageConsumer consumer = jmsContext.createConsumer(session, slaListener.getTopic(startMiss));
slaListener.onStartMiss(startMiss);
TextMessage message = (TextMessage) consumer.receive(5000);
SLAMessage slaStartMissMsg = (SLAMessage) JMSMessagingUtils.getEventMessage(message);
// check msg header
assertEquals(EventStatus.START_MISS, slaStartMissMsg.getEventStatus());
assertEquals(SLAStatus.NOT_STARTED, slaStartMissMsg.getSLAStatus());
assertEquals(AppType.COORDINATOR_ACTION, slaStartMissMsg.getAppType());
assertEquals(MessageType.SLA, slaStartMissMsg.getMessageType());
assertEquals("Test-SLA-Start-Miss", slaStartMissMsg.getAppName());
assertEquals("dummyuser", slaStartMissMsg.getUser());
// check msg body
assertEquals("0000000-000000000000001-oozie-wrkf-C@1", slaStartMissMsg.getId());
assertEquals("0000000-000000000000001-oozie-wrkf-C", slaStartMissMsg.getParentId());
assertEquals(startDate, slaStartMissMsg.getExpectedStartTime());
assertEquals("notification of start miss", slaStartMissMsg.getNotificationMessage());
}
public void testOnSLAEndMissEvent() throws Exception {
JMSSLAEventListener slaListener = new JMSSLAEventListener();
slaListener.init(conf);
SLACalcStatus endMiss = new SLACalcStatus(new SLARegistrationBean());
SLARegistrationBean endMissBean = endMiss.getSLARegistrationBean();
Date expectedEndDate = DateUtils.parseDateUTC("2013-01-01T00:00Z");
Date actualEndDate = DateUtils.parseDateUTC("2013-01-01T01:00Z");
endMiss.setId("0000000-000000000000001-oozie-wrkf-C@1");
endMissBean.setParentId("0000000-000000000000001-oozie-wrkf-C");
endMissBean.setAppName("Test-SLA-End-Miss");
endMiss.setEventStatus(EventStatus.END_MISS);
endMiss.setSLAStatus(SLAStatus.IN_PROCESS);
endMissBean.setAppType(AppType.COORDINATOR_ACTION);
endMissBean.setUser("dummyuser");
endMissBean.setNotificationMsg("notification of end miss");
endMissBean.setExpectedEnd(expectedEndDate);
endMiss.setActualEnd(actualEndDate);
ConnectionContext jmsContext = getConnectionContext();
Session session = jmsContext.createSession(Session.AUTO_ACKNOWLEDGE);
MessageConsumer consumer = jmsContext.createConsumer(session, slaListener.getTopic(endMiss));
slaListener.onEndMiss(endMiss);
TextMessage message = (TextMessage) consumer.receive(5000);
SLAMessage slaEndMissMsg = (SLAMessage) JMSMessagingUtils.getEventMessage(message);
// check msg header
assertEquals(EventStatus.END_MISS, slaEndMissMsg.getEventStatus());
assertEquals(SLAStatus.IN_PROCESS, slaEndMissMsg.getSLAStatus());
assertEquals(AppType.COORDINATOR_ACTION, slaEndMissMsg.getAppType());
assertEquals(MessageType.SLA, slaEndMissMsg.getMessageType());
assertEquals("Test-SLA-End-Miss", slaEndMissMsg.getAppName());
assertEquals("dummyuser", slaEndMissMsg.getUser());
// check msg body
assertEquals("0000000-000000000000001-oozie-wrkf-C@1", slaEndMissMsg.getId());
assertEquals("0000000-000000000000001-oozie-wrkf-C", slaEndMissMsg.getParentId());
assertEquals(expectedEndDate, slaEndMissMsg.getExpectedEndTime());
assertEquals(actualEndDate, slaEndMissMsg.getActualEndTime());
assertEquals("notification of end miss", slaEndMissMsg.getNotificationMessage());
}
/** @return the initial instance of a DataSet in Calendar */
private static Calendar getInitialInstanceCal() {
ELEvaluator eval = ELEvaluator.getCurrent();
SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
if (ds == null) {
throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
}
Calendar effInitTS = Calendar.getInstance();
effInitTS.setTime(ds.getInitInstance());
effInitTS.setTimeZone(ds.getTimeZone());
// To adjust EOD/EOM
DateUtils.moveToEnd(effInitTS, getDSEndOfFlag());
return effInitTS;
// return ds.getInitInstance();
}
/**
* Determine how many hours is on the date of n-th dataset instance.
*
* <p>It depends on:
*
* <p>1. Data set frequency
*
* <p>2. Data set Time unit (day, month, minute)
*
* <p>3. Data set Time zone/DST
*
* <p>4. End Day/Month flag
*
* <p>5. Data set initial instance
*
* <p>6. Action Creation Time
*
* @param n instance count
* <p>domain: n is integer
* @return number of hours on that day
* <p>returns -1 means n-th instance is earlier than Initial-Instance of DS
* @throws Exception
*/
public static int ph2_coord_hoursInDay(int n) throws Exception {
int datasetFrequency = (int) getDSFrequency();
// /Calendar nominalInstanceCal =
// getCurrentInstance(getActionCreationtime());
Calendar nominalInstanceCal = getEffectiveNominalTime();
if (nominalInstanceCal == null) {
return -1;
}
nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n);
/*
* if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0)
* { return -1; }
*/
nominalInstanceCal.setTimeZone(getDatasetTZ()); // Use Dataset TZ
// DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
return DateUtils.hoursInDay(nominalInstanceCal);
}
/**
* @param n
* @return n-th instance Date-Time from current instance for data-set
* <p>return empty string ("") if the Action_Creation_time or the n-th instance
* <p>is earlier than the Initial_Instance of dataset.
* @throws Exception
*/
private static String coord_current_sync(int n) throws Exception {
int datasetFrequency = getDSFrequency(); // in minutes
TimeUnit dsTimeUnit = getDSTimeUnit();
int[] instCount = new int[1]; // used as pass by ref
Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
if (nominalInstanceCal == null) {
return "";
}
nominalInstanceCal = getInitialInstanceCal();
int absInstanceCount = instCount[0] + n;
nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency * absInstanceCount);
if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) {
return "";
}
String str = DateUtils.formatDateUTC(nominalInstanceCal);
return str;
}
@Test
public void testCoordOffset() throws Exception {
CoordinatorJobBean job =
addRecordToCoordJobTableForWaiting(
"coord-dataset-offset.xml", CoordinatorJob.Status.RUNNING, false, true);
Path appPath = new Path(getFsTestCaseDir(), "coord");
String actionXml = getCoordActionXml(appPath, "coord-dataset-offset.xml");
actionXml = actionXml.replace("-unit-", "DAY");
actionXml = actionXml.replace("-frequency-", "1");
CoordinatorActionBean actionBean = createCoordinatorActionBean(job);
Configuration jobConf = new XConfiguration(new StringReader(job.getConf()));
Element eAction = createActionElement(actionXml);
jobConf.set("startInstance", "coord:offset(-4,DAY)");
jobConf.set("endInstance", "coord:offset(0,DAY)");
String output =
CoordCommandUtils.materializeOneInstance(
"jobId",
true,
eAction,
DateUtils.parseDateOozieTZ("2009-08-20T10:00Z"),
DateUtils.parseDateOozieTZ("2009-08-20T10:00Z"),
1,
jobConf,
actionBean);
eAction = XmlUtils.parseXml(output);
Element e =
(Element)
((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0))
.getChildren()
.get(0);
assertEquals(
e.getChild("uris", e.getNamespace()).getTextTrim(),
"hdfs:///tmp/workflows/2009/08/20/01;region=us#hdfs:///tmp/workflows/2009/08/19/01;region=us#"
+ "hdfs:///tmp/workflows/2009/08/18/01;region=us#hdfs:///tmp/workflows/2009/08/17/01;"
+ "region=us#hdfs:///tmp/workflows/2009/08/16/01;region=us");
jobConf.set("startInstance", "coord:offset(-4,HOUR)");
jobConf.set("endInstance", "coord:offset(0,HOUR)");
actionXml = getCoordActionXml(appPath, "coord-dataset-offset.xml");
actionXml = actionXml.replace("-unit-", "MINUTE");
actionXml = actionXml.replace("-frequency-", "60");
eAction = createActionElement(actionXml);
output =
CoordCommandUtils.materializeOneInstance(
"jobId",
true,
eAction,
DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"),
DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"),
1,
jobConf,
actionBean);
eAction = XmlUtils.parseXml(output);
e =
(Element)
((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0))
.getChildren()
.get(0);
assertEquals(
e.getChild("uris", e.getNamespace()).getTextTrim(),
"hdfs:///tmp/workflows/2009/08/20/01;region=us#hdfs:///tmp/workflows/2009/08/20/00;region=us#"
+ "hdfs:///tmp/workflows/2009/08/19/23;region=us#hdfs:///tmp/workflows/2009/08/19/22;region=us#"
+ "hdfs:///tmp/workflows/2009/08/19/21;region=us");
}
@Test
public void testCoordAbsolute() throws Exception {
CoordinatorJobBean job =
addRecordToCoordJobTableForWaiting(
"coord-dataset-absolute.xml", CoordinatorJob.Status.RUNNING, false, true);
Path appPath = new Path(getFsTestCaseDir(), "coord");
String actionXml = getCoordActionXml(appPath, "coord-dataset-absolute.xml");
CoordinatorActionBean actionBean = createCoordinatorActionBean(job);
Configuration jobConf = new XConfiguration(new StringReader(job.getConf()));
Element eAction = createActionElement(actionXml);
jobConf.set("startInstance", "coord:absolute(2009-08-20T01:00Z)");
jobConf.set("endInstance", "coord:current(2)");
String output =
CoordCommandUtils.materializeOneInstance(
"jobId",
true,
eAction,
DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"),
DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"),
1,
jobConf,
actionBean);
eAction = XmlUtils.parseXml(output);
List<?> elementList =
((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0))
.getChildren();
Element e1 = (Element) elementList.get(0);
Element e2 = (Element) elementList.get(1);
// startInstance = coord:absolute(2009-08-20T01:00Z) which is current(0)
// and endInstance = coord:current(2).
assertEquals(
e1.getChild("uris", e1.getNamespace()).getTextTrim(),
"hdfs:///tmp/workflows/2009/09/03;region=us#hdfs:///tmp/workflows/2009/08/27;"
+ "region=us#hdfs:///tmp/workflows/2009/08/20;region=us");
// Test parameterized with startInstance =
// coord:absolute(2009-08-20T01:00Z) which is current (0) and
// endInstance = coord:current(2)
assertEquals(
e2.getChild("uris", e1.getNamespace()).getTextTrim(),
"hdfs:///tmp/workflows/2009/09/03;region=us#hdfs:///tmp/workflows/2009/08/27;"
+ "region=us#hdfs:///tmp/workflows/2009/08/20;region=us");
// Test when start instance < nominal time. 2009-08-20T01:00Z is
// current(-3)
jobConf.set("startInstance", "coord:absolute(2009-08-20T01:00Z)");
jobConf.set("endInstance", "coord:current(2)");
eAction = createActionElement(actionXml);
output =
CoordCommandUtils.materializeOneInstance(
"jobId",
true,
eAction,
DateUtils.parseDateOozieTZ("2009-09-10T01:00Z"),
DateUtils.parseDateOozieTZ("2009-09-08T01:00Z"),
1,
jobConf,
actionBean);
eAction = XmlUtils.parseXml(output);
elementList =
((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0))
.getChildren();
e1 = (Element) elementList.get(1);
assertEquals(
e1.getChild("uris", e1.getNamespace()).getTextTrim(),
"hdfs:///tmp/workflows/2009/09/24;region=us#hdfs:///tmp/workflows/2009/09/17;region=us#"
+ "hdfs:///tmp/workflows/2009/09/10;region=us#hdfs:///tmp/workflows/2009/09/03;region=us#"
+ "hdfs:///tmp/workflows/2009/08/27;region=us#hdfs:///tmp/workflows/2009/08/20;region=us");
// // Test when start instance > nominal time. 2009-08-20T01:00Z is
// current(1)
jobConf.set("startInstance", "coord:absolute(2009-08-20T01:00Z)");
jobConf.set("endInstance", "coord:current(2)");
eAction = createActionElement(actionXml);
output =
CoordCommandUtils.materializeOneInstance(
"jobId",
true,
eAction,
DateUtils.parseDateOozieTZ("2009-08-14T01:00Z"),
DateUtils.parseDateOozieTZ("2009-08-14T01:00Z"),
1,
jobConf,
actionBean);
eAction = XmlUtils.parseXml(output);
elementList =
((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0))
.getChildren();
e1 = (Element) elementList.get(1);
assertEquals(
e1.getChild("uris", e1.getNamespace()).getTextTrim(),
"hdfs:///tmp/workflows/2009/08/27;region=us#hdfs:///tmp/workflows/2009/08/20;region=us");
try {
// Test start instance > end instance. start = 2009-08-27T01:00Z
// which is current(3)
jobConf.set("startInstance", "coord:absolute(2009-08-27T01:00Z)");
jobConf.set("endInstance", "coord:current(2)");
eAction = createActionElement(actionXml);
output =
CoordCommandUtils.materializeOneInstance(
"jobId",
true,
eAction,
DateUtils.parseDateOozieTZ("2009-08-06T01:00Z"),
DateUtils.parseDateOozieTZ("2009-08-16T01:00Z"),
1,
jobConf,
actionBean);
eAction = XmlUtils.parseXml(output);
fail("Should throw exception. Start-instance > end-instance ");
} catch (Exception e) {
assertTrue(
e.getCause()
.getMessage()
.contains("start-instance should be equal or earlier than the end-instance"));
}
try {
// Test start instance < initial instance. initial instance =
// 2009-08-06T01:00Z and start = 2009-07-01T01:00Z
jobConf.set("startInstance", "coord:absolute(2009-07-01T01:00Z)");
jobConf.set("endInstance", "coord:current(2)");
eAction = createActionElement(actionXml);
output =
CoordCommandUtils.materializeOneInstance(
"jobId",
true,
eAction,
DateUtils.parseDateOozieTZ("2009-08-06T01:00Z"),
DateUtils.parseDateOozieTZ("2009-08-16T01:00Z"),
1,
jobConf,
actionBean);
eAction = XmlUtils.parseXml(output);
fail("Should throw exception. Start-instance > end-instance ");
} catch (Exception e) {
assertTrue(
e.getCause()
.getMessage()
.contains("intial-instance should be equal or earlier than the start-instance"));
}
try {
// Test Exception. Start-instance as absolute and end-instance as
// latest.
jobConf.set("startInstance", "coord:absolute(2009-08-20T01:00Z)");
jobConf.set("endInstance", "coord:latest(2)");
eAction = createActionElement(actionXml);
output =
CoordCommandUtils.materializeOneInstance(
"jobId",
true,
eAction,
DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"),
DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"),
1,
jobConf,
actionBean);
eAction = XmlUtils.parseXml(output);
fail("Should throw exception. Start-instance is absolute and end-instance is latest");
} catch (Exception e) {
assertTrue(
e.getMessage()
.contains(
"Only start-instance as absolute and end-instance as current is supported"));
}
}
/**
* Resolve basic entities using job Configuration.
*
* @param conf :Job configuration
* @param appXml : Original job XML
* @param coordJob : Coordinator job bean to be populated.
* @return Resolved job XML element.
* @throws CoordinatorJobException thrown if failed to resolve basic entities
* @throws Exception thrown if failed to resolve basic entities
*/
@SuppressWarnings("unchecked")
protected Element resolveInitial(Configuration conf, String appXml, CoordinatorJobBean coordJob)
throws CoordinatorJobException, Exception {
Element eAppXml = XmlUtils.parseXml(appXml);
// job's main attributes
// frequency
String val = resolveAttribute("frequency", eAppXml, evalFreq);
int ival = ParamChecker.checkInteger(val, "frequency");
ParamChecker.checkGTZero(ival, "frequency");
coordJob.setFrequency(ival);
TimeUnit tmp =
(evalFreq.getVariable("timeunit") == null)
? TimeUnit.MINUTE
: ((TimeUnit) evalFreq.getVariable("timeunit"));
addAnAttribute("freq_timeunit", eAppXml, tmp.toString());
// TimeUnit
coordJob.setTimeUnit(CoordinatorJob.Timeunit.valueOf(tmp.toString()));
// End Of Duration
tmp =
evalFreq.getVariable("endOfDuration") == null
? TimeUnit.NONE
: ((TimeUnit) evalFreq.getVariable("endOfDuration"));
addAnAttribute("end_of_duration", eAppXml, tmp.toString());
// coordJob.setEndOfDuration(tmp) // TODO: Add new attribute in Job bean
// Application name
if (this.coordName == null) {
val = resolveAttribute("name", eAppXml, evalNofuncs);
coordJob.setAppName(val);
} else {
// this coord job is created from bundle
coordJob.setAppName(this.coordName);
}
// start time
val = resolveAttribute("start", eAppXml, evalNofuncs);
ParamChecker.checkUTC(val, "start");
coordJob.setStartTime(DateUtils.parseDateUTC(val));
// end time
val = resolveAttribute("end", eAppXml, evalNofuncs);
ParamChecker.checkUTC(val, "end");
coordJob.setEndTime(DateUtils.parseDateUTC(val));
// Time zone
val = resolveAttribute("timezone", eAppXml, evalNofuncs);
ParamChecker.checkTimeZone(val, "timezone");
coordJob.setTimeZone(val);
// controls
val =
resolveTagContents(
"timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
if (val == "") {
val = Services.get().getConf().get(CONF_DEFAULT_TIMEOUT_NORMAL);
}
ival = ParamChecker.checkInteger(val, "timeout");
if (ival < 0 || ival > Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600)) {
ival = Services.get().getConf().getInt(CONF_DEFAULT_MAX_TIMEOUT, 129600);
}
coordJob.setTimeout(ival);
val =
resolveTagContents(
"concurrency", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
if (val == null || val.isEmpty()) {
val = Services.get().getConf().get(CONF_DEFAULT_CONCURRENCY, "-1");
}
ival = ParamChecker.checkInteger(val, "concurrency");
coordJob.setConcurrency(ival);
val =
resolveTagContents(
"throttle", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
if (val == null || val.isEmpty()) {
int defaultThrottle = Services.get().getConf().getInt(CONF_DEFAULT_THROTTLE, 12);
ival = defaultThrottle;
} else {
ival = ParamChecker.checkInteger(val, "throttle");
}
int maxQueue = Services.get().getConf().getInt(CONF_QUEUE_SIZE, 10000);
float factor = Services.get().getConf().getFloat(CONF_MAT_THROTTLING_FACTOR, 0.10f);
int maxThrottle = (int) (maxQueue * factor);
if (ival > maxThrottle || ival < 1) {
ival = maxThrottle;
}
LOG.debug("max throttle " + ival);
coordJob.setMatThrottling(ival);
val =
resolveTagContents(
"execution", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
if (val == "") {
val = Execution.FIFO.toString();
}
coordJob.setExecution(Execution.valueOf(val));
String[] acceptedVals = {
Execution.LIFO.toString(), Execution.FIFO.toString(), Execution.LAST_ONLY.toString()
};
ParamChecker.isMember(val, acceptedVals, "execution");
// datasets
resolveTagContents(
"include", eAppXml.getChild("datasets", eAppXml.getNamespace()), evalNofuncs);
// for each data set
resolveDataSets(eAppXml);
HashMap<String, String> dataNameList = new HashMap<String, String>();
resolveIOEvents(eAppXml, dataNameList);
resolveTagContents(
"app-path",
eAppXml
.getChild("action", eAppXml.getNamespace())
.getChild("workflow", eAppXml.getNamespace()),
evalNofuncs);
// TODO: If action or workflow tag is missing, NullPointerException will
// occur
Element configElem =
eAppXml
.getChild("action", eAppXml.getNamespace())
.getChild("workflow", eAppXml.getNamespace())
.getChild("configuration", eAppXml.getNamespace());
evalData =
CoordELEvaluator.createELEvaluatorForDataEcho(conf, "coord-job-submit-data", dataNameList);
if (configElem != null) {
for (Element propElem :
(List<Element>) configElem.getChildren("property", configElem.getNamespace())) {
resolveTagContents("name", propElem, evalData);
// Want to check the data-integrity but don't want to modify the
// XML
// for properties only
Element tmpProp = (Element) propElem.clone();
resolveTagContents("value", tmpProp, evalData);
}
}
resolveSLA(eAppXml, coordJob);
return eAppXml;
}
@Test
public void testComputeNextNominalTime() throws Exception {
// The 10th minute of every hour
CoordinatorJobBean job = new CoordinatorJobBean();
CoordinatorActionBean action = new CoordinatorActionBean();
job.setFrequency("10 * * * *");
job.setTimeUnit(CoordinatorJob.Timeunit.CRON);
job.setTimeZone("America/Los_Angeles");
job.setStartTime(DateUtils.parseDateOozieTZ("2015-07-21T08:00Z"));
job.setEndTime(DateUtils.parseDateOozieTZ("2015-07-21T13:50Z"));
job.setJobXml("<coordinator-app name=\"test\" end_of_duration='NONE'></coordinator-app>");
// @5 occurs at the 10th minute of the 5th hour after the starting time (12:10)
action.setNominalTime(DateUtils.parseDateOozieTZ("2015-07-21T12:10Z"));
action.setActionNumber(5);
// The next nominal time should be at the 10th minute of the next hour (13:10)
assertEquals(
DateUtils.parseDateOozieTZ("2015-07-21T13:10Z"),
CoordCommandUtils.computeNextNominalTime(job, action));
job.setEndTime(DateUtils.parseDateOozieTZ("2015-07-21T13:00Z"));
assertNull(CoordCommandUtils.computeNextNominalTime(job, action));
// Every 10 hours
job = new CoordinatorJobBean();
action = new CoordinatorActionBean();
job.setFrequency("10");
job.setTimeUnit(CoordinatorJob.Timeunit.HOUR);
job.setTimeZone("America/Los_Angeles");
job.setStartTime(DateUtils.parseDateOozieTZ("2015-07-21T14:00Z"));
job.setEndTime(DateUtils.parseDateOozieTZ("2015-07-25T23:00Z"));
job.setJobXml("<coordinator-app name=\"test\" end_of_duration='NONE'></coordinator-app>");
// @5 occurs 50 hours after the start time (23T06:00)
action.setNominalTime(DateUtils.parseDateOozieTZ("2015-07-23T06:00Z"));
action.setActionNumber(5);
// The next nominal time should be 10 hours after that (23T16:00)
assertEquals(
DateUtils.parseDateOozieTZ("2015-07-23T16:00Z"),
CoordCommandUtils.computeNextNominalTime(job, action));
job.setEndTime(DateUtils.parseDateOozieTZ("2015-07-23T09:00Z"));
assertNull(CoordCommandUtils.computeNextNominalTime(job, action));
// Every 10 days, at the end of the day
job = new CoordinatorJobBean();
action = new CoordinatorActionBean();
job.setFrequency("10");
job.setTimeUnit(CoordinatorJob.Timeunit.END_OF_DAY);
job.setTimeZone("America/Los_Angeles");
job.setStartTime(DateUtils.parseDateOozieTZ("2015-07-21T14:00Z"));
job.setEndTime(DateUtils.parseDateOozieTZ("2015-09-21T23:00Z"));
job.setJobXml("<coordinator-app name=\"test\" end_of_duration='END_OF_DAY'></coordinator-app>");
// @5 occurs at 50 days, at the end of the day, after the starting time (08-31T07:00Z)
// (Timezone is America/Los_Angeles so end of day is -0700)
action.setNominalTime(DateUtils.parseDateOozieTZ("2015-08-31T07:00Z"));
action.setActionNumber(5);
// The next nominal time should be 10 days after that, at the end of the day
assertEquals(
DateUtils.parseDateOozieTZ("2015-09-10T07:00Z"),
CoordCommandUtils.computeNextNominalTime(job, action));
job.setEndTime(DateUtils.parseDateOozieTZ("2015-09-08T23:00Z"));
assertNull(CoordCommandUtils.computeNextNominalTime(job, action));
}
public void testOffset() throws Exception {
init("coord-action-create");
String expr = "${coord:offset(-1440, \"MINUTE\")}";
assertEquals("2009-09-08T23:59Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:offset(-24, \"HOUR\")}";
assertEquals("2009-09-08T23:59Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:offset(-1, \"DAY\")}";
assertEquals("2009-09-08T23:59Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:offset(1, \"MONTH\")}";
assertEquals("2009-10-09T23:59Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:offset(1, \"YEAR\")}";
assertEquals("2010-09-09T23:59Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:offset(-10, \"DAY\")}";
assertEquals("", CoordELFunctions.evalAndWrap(eval, expr));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2015-01-02T00:45Z"));
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.YEAR);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2010-01-02T00:01Z"));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
// Year
expr =
"${coord:offset(0, \"YEAR\")} ${coord:offset(1, \"YEAR\")} ${coord:offset(-1, \"YEAR\")}"
+ " ${coord:offset(-3, \"YEAR\")}";
assertEquals(
"2015-01-02T00:01Z 2016-01-02T00:01Z 2014-01-02T00:01Z 2012-01-02T00:01Z",
eval.evaluate(expr, String.class));
// Month
expr =
"${coord:offset(0, \"MONTH\")} ${coord:offset(12, \"MONTH\")} ${coord:offset(-12, \"MONTH\")}"
+ " ${coord:offset(-36, \"MONTH\")}";
assertEquals(
"2015-01-02T00:01Z 2016-01-02T00:01Z 2014-01-02T00:01Z 2012-01-02T00:01Z",
eval.evaluate(expr, String.class));
// Day
// its -1096 instead of -1095 because of DST (extra 1 day)
expr =
"${coord:offset(0, \"DAY\")} ${coord:offset(365, \"DAY\")} ${coord:offset(-365, \"DAY\")}"
+ " ${coord:offset(-1096, \"DAY\")}";
assertEquals(
"2015-01-02T00:01Z 2016-01-02T00:01Z 2014-01-02T00:01Z 2012-01-02T00:01Z",
eval.evaluate(expr, String.class));
// Hour
// its -26304 instead of -26280 because of DST (extra 24 hours)
expr =
"${coord:offset(0, \"HOUR\")} ${coord:offset(8760, \"HOUR\")} ${coord:offset(-8760, \"HOUR\")}"
+ " ${coord:offset(-26304, \"HOUR\")}";
assertEquals(
"2015-01-02T00:01Z 2016-01-02T00:01Z 2014-01-02T00:01Z 2012-01-02T00:01Z",
eval.evaluate(expr, String.class));
// Minute
// its -1578240 instead of -1576800 because of DST (extra 1440 minutes)
expr =
"${coord:offset(0, \"MINUTE\")} ${coord:offset(525600, \"MINUTE\")} ${coord:offset(-525600, \"MINUTE\")}"
+ " ${coord:offset(-1578240, \"MINUTE\")}";
assertEquals(
"2015-01-02T00:01Z 2016-01-02T00:01Z 2014-01-02T00:01Z 2012-01-02T00:01Z",
eval.evaluate(expr, String.class));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2015-01-02T00:45Z"));
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.MINUTE);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2010-01-02T00:01Z"));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
// Minute
expr =
"${coord:offset(0, \"MINUTE\")} ${coord:offset(1, \"MINUTE\")} ${coord:offset(-1, \"MINUTE\")}"
+ " ${coord:offset(-3, \"MINUTE\")}";
assertEquals(
"2015-01-02T00:45Z 2015-01-02T00:46Z 2015-01-02T00:44Z 2015-01-02T00:42Z",
eval.evaluate(expr, String.class));
// Hour
expr =
"${coord:offset(0, \"HOUR\")} ${coord:offset(1, \"HOUR\")} ${coord:offset(-1, \"HOUR\")}"
+ " ${coord:offset(-3, \"HOUR\")}";
assertEquals(
"2015-01-02T00:45Z 2015-01-02T01:45Z 2015-01-01T23:45Z 2015-01-01T21:45Z",
eval.evaluate(expr, String.class));
// Day
expr =
"${coord:offset(0, \"DAY\")} ${coord:offset(1, \"DAY\")} ${coord:offset(-1, \"DAY\")}"
+ " ${coord:offset(-3, \"DAY\")}";
assertEquals(
"2015-01-02T00:45Z 2015-01-03T00:45Z 2015-01-01T00:45Z 2014-12-30T00:45Z",
eval.evaluate(expr, String.class));
// Month
expr =
"${coord:offset(0, \"MONTH\")} ${coord:offset(1, \"MONTH\")} ${coord:offset(-1, \"MONTH\")}"
+ " ${coord:offset(-3, \"MONTH\")}";
assertEquals(
"2015-01-02T00:45Z 2015-02-02T00:45Z 2014-12-02T00:45Z 2014-10-01T23:45Z",
eval.evaluate(expr, String.class));
// Year
expr =
"${coord:offset(0, \"YEAR\")} ${coord:offset(1, \"YEAR\")} ${coord:offset(-1, \"YEAR\")}"
+ " ${coord:offset(-3, \"YEAR\")}";
assertEquals(
"2015-01-02T00:45Z 2016-01-02T00:45Z 2014-01-02T00:45Z 2012-01-02T00:45Z",
eval.evaluate(expr, String.class));
// Test rewinding when the given offset isn't a multiple of the
// frequency
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2015-01-02T00:45Z"));
ds.setFrequency(4);
ds.setTimeUnit(TimeUnit.HOUR);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2010-01-02T00:01Z"));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
expr = "${coord:offset(5, \"MINUTE\")}";
assertEquals("2015-01-02T00:01Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:offset(1, \"HOUR\")}";
assertEquals("2015-01-02T00:01Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:offset(7, \"HOUR\")}";
assertEquals("2015-01-02T04:01Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:offset(-2, \"HOUR\")}";
assertEquals("2015-01-01T20:01Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:offset(-43825, \"HOUR\")}";
assertEquals("", CoordELFunctions.evalAndWrap(eval, expr));
// "blah" is not a valid TimeUnit
expr = "${coord:offset(1, \"blah\")}";
try {
CoordELFunctions.evalAndWrap(eval, expr);
fail("eval of " + expr + " should have thrown an exception");
} catch (Exception e) {
assertTrue(e.getMessage().contains("Unable to evaluate"));
}
// 4.5 is not a valid integer
expr = "${coord:offset(4.5, \"blah\")}";
try {
CoordELFunctions.evalAndWrap(eval, expr);
fail("eval of " + expr + " should have thrown an exception");
} catch (Exception e) {
assertTrue(e.getMessage().contains("Unable to evaluate"));
}
}
public void testCurrent() throws Exception {
init("coord-action-create");
String expr = "${coord:current(-1)}";
assertEquals("2009-09-08T23:59Z", CoordELFunctions.evalAndWrap(eval, expr));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-05-30T00:45Z"));
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.DAY);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-02T00:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
expr = "${coord:current(0)} ${coord:current(1)} ${coord:current(-1)} ${coord:current(-3)}";
assertEquals(
"2009-05-29T23:00Z 2009-05-30T23:00Z 2009-05-28T23:00Z 2009-05-26T23:00Z",
CoordELFunctions.evalAndWrap(eval, expr));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-05-30T00:45Z"));
ds.setFrequency(30);
ds.setTimeUnit(TimeUnit.MINUTE);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-08T00:00Z"));
expr = "${coord:current(0)} ${coord:current(1)} ${coord:current(-1)} ${coord:current(-3)}";
assertEquals(
"2009-05-30T00:30Z 2009-05-30T01:00Z 2009-05-30T00:00Z 2009-05-29T23:00Z",
eval.evaluate(expr, String.class));
SyncCoordAction appInst = new SyncCoordAction();
SyncCoordDataset ds = new SyncCoordDataset();
;
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.DAY);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-02T00:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setName("test1");
ds.setType("SYNC");
appInst.setActualTime(DateUtils.parseDateOozieTZ("2009-09-10T23:59Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-05-30T00:00Z "));
appInst.setTimeZone(DateUtils.getTimeZone("UTC"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-05-30T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-05-31T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(-1)}";
assertEquals("2009-05-29T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(-3)}";
assertEquals("2009-05-27T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
ds.setFrequency(7);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-08T00:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-05-28T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-06-04T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(-1)}";
assertEquals("2009-05-21T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(-3)}";
assertEquals("2009-05-07T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Spring DST transition
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-08T10:45Z"));
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.DAY);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-03-06T10:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
expr =
"${coord:current(-2)} ${coord:current(-1)} ${coord:current(0)} ${coord:current(1)} ${coord:current(2)}";
assertEquals(
"2009-03-06T10:00Z 2009-03-07T10:00Z 2009-03-08T09:00Z 2009-03-09T09:00Z 2009-03-10T09:00Z",
CoordELFunctions.evalAndWrap(eval, expr));
// Winter DST Transition
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-11-01T08:00Z"));
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.DAY);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-10-30T08:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
expr =
"${coord:current(-2)} ${coord:current(-1)} ${coord:current(0)} ${coord:current(1)} ${coord:current(2)}";
// System.out.println("AAAAA " + CoordELFunctions.evalAndWrap(eval,
// expr));
assertEquals(
"2009-10-30T08:00Z 2009-10-31T08:00Z 2009-11-01T08:00Z 2009-11-02T09:00Z 2009-11-03T09:00Z",
CoordELFunctions.evalAndWrap(eval, expr));
// EndofDay testing
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.DAY);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-02T09:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setName("test1");
ds.setType("SYNC");
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-05-30T12:00Z "));
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-05-30T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-05-31T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// From Specification
// Case 1
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T08:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-01-01T08:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("", CoordELFunctions.evalAndWrap(eval, expr));
// Case 2
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T08:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-01-01T08:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
// assertEquals("2009-01-02T08:00Z", CoordELFunctions.evalAndWrap(eval,
// expr));
assertEquals("", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
// assertEquals("2009-01-03T08:00Z", CoordELFunctions.evalAndWrap(eval,
// expr));
assertEquals("", CoordELFunctions.evalAndWrap(eval, expr));
// Case 3
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T07:01Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-01-01T08:01Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-01-01T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-01-02T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 4
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T7:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-01-01T18:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-01-01T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-01-02T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 5
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-03-07T07:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-07T09:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-03-07T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-03-08T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 6
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-03-08T07:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-08T08:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-03-08T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-03-09T07:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 7
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-03-09T07:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-10T08:01Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-03-10T07:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-03-11T07:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 8
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
ds.setFrequency(2); // Changed
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-03-09T07:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-10T07:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-03-10T07:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-03-12T07:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Test with EOM
ds.setTimeUnit(TimeUnit.MONTH);
// Case 1
ds.setEndOfDuration(TimeUnit.END_OF_MONTH);
ds.setFrequency(1);
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T00:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-02-01T00:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-02-01T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-03-01T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 2
ds.setEndOfDuration(TimeUnit.END_OF_MONTH);
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T08:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-02-01T08:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-02-01T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-03-01T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 3
ds.setEndOfDuration(TimeUnit.END_OF_MONTH);
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-31T08:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-02-01T08:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-02-01T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-03-01T00:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 4
ds.setEndOfDuration(TimeUnit.END_OF_MONTH);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T08:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-02-01T08:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-02-01T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-03-01T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 5
ds.setEndOfDuration(TimeUnit.END_OF_MONTH);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-02-02T08:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-02T08:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-03-01T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-04-01T07:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 6
ds.setEndOfDuration(TimeUnit.END_OF_MONTH);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-02-01T08:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-01T08:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-03-01T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-04-01T07:00Z", CoordELFunctions.evalAndWrap(eval, expr));
// Case 7
ds.setEndOfDuration(TimeUnit.END_OF_MONTH);
ds.setFrequency(3);
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-02-01T08:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-01T08:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:current(0)}";
assertEquals("2009-03-01T08:00Z", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:current(1)}";
assertEquals("2009-06-01T07:00Z", CoordELFunctions.evalAndWrap(eval, expr));
}
public static SLARegistrationBean createSlaRegistrationEvent(
Element eSla,
String jobId,
String parentId,
AppType appType,
String user,
String appName,
XLog log,
boolean rerun)
throws CommandException {
if (eSla == null || !SLAService.isEnabled()) {
log.debug("Not registering SLA for job [{0}]. Sla-Xml null OR SLAService not enabled", jobId);
return null;
}
SLARegistrationBean sla = new SLARegistrationBean();
// Setting nominal time
String strNominalTime = getTagElement(eSla, NOMINAL_TIME);
if (strNominalTime == null || strNominalTime.length() == 0) {
throw new CommandException(ErrorCode.E1101, NOMINAL_TIME);
}
Date nominalTime;
try {
nominalTime = DateUtils.parseDateOozieTZ(strNominalTime);
sla.setNominalTime(nominalTime);
} catch (ParseException pex) {
throw new CommandException(ErrorCode.E0302, strNominalTime, pex);
}
// Setting expected start time
String strExpectedStart = getTagElement(eSla, SHOULD_START);
if (strExpectedStart != null) {
float expectedStart = Float.parseFloat(strExpectedStart);
if (expectedStart < 0) {
throw new CommandException(
ErrorCode.E0302, strExpectedStart, "for SLA Expected start time");
} else {
Date expectedStartTime =
new Date(nominalTime.getTime() + (long) (expectedStart * 60 * 1000));
sla.setExpectedStart(expectedStartTime);
}
}
// Setting expected end time
String strExpectedEnd = getTagElement(eSla, SHOULD_END);
if (strExpectedEnd == null || strExpectedEnd.length() == 0) {
throw new CommandException(ErrorCode.E1101, SHOULD_END);
}
float expectedEnd = Float.parseFloat(strExpectedEnd);
if (expectedEnd < 0) {
throw new CommandException(ErrorCode.E0302, strExpectedEnd, "for SLA Expected end time");
} else {
Date expectedEndTime = new Date(nominalTime.getTime() + (long) (expectedEnd * 60 * 1000));
sla.setExpectedEnd(expectedEndTime);
}
// Setting expected duration in milliseconds
String expectedDurationStr = getTagElement(eSla, MAX_DURATION);
if (expectedDurationStr != null && expectedDurationStr.length() > 0) {
float expectedDuration = Float.parseFloat(expectedDurationStr);
if (expectedDuration > 0) {
sla.setExpectedDuration((long) (expectedDuration * 60 * 1000));
}
} else if (sla.getExpectedStart() != null) {
sla.setExpectedDuration(sla.getExpectedEnd().getTime() - sla.getExpectedStart().getTime());
}
// Parse desired alert-types i.e. start-miss, end-miss, start-met etc..
String alertEvents = getTagElement(eSla, ALERT_EVENTS);
if (alertEvents != null) {
String events[] = alertEvents.split(",");
StringBuilder alertsStr = new StringBuilder();
for (int i = 0; i < events.length; i++) {
String event = events[i].trim().toUpperCase();
try {
EventStatus.valueOf(event);
} catch (IllegalArgumentException iae) {
XLog.getLog(SLAService.class)
.warn(
"Invalid value: ["
+ event
+ "]"
+ " for SLA Alert-event. Should be one of "
+ EventStatus.values()
+ ". Setting it to default ["
+ EventStatus.END_MISS.name()
+ "]");
event = EventStatus.END_MISS.name();
}
alertsStr.append(event).append(",");
}
sla.setAlertEvents(alertsStr.toString().substring(0, alertsStr.lastIndexOf(",")));
}
// Other sla config
sla.setNotificationMsg(getTagElement(eSla, "notification-msg"));
sla.setAlertContact(getTagElement(eSla, "alert-contact"));
sla.setUpstreamApps(getTagElement(eSla, "upstream-apps"));
// Oozie defined
sla.setId(jobId);
sla.setAppType(appType);
sla.setAppName(appName);
sla.setUser(user);
sla.setParentId(parentId);
SLAService slaService = Services.get().get(SLAService.class);
try {
if (!rerun) {
slaService.addRegistrationEvent(sla);
} else {
slaService.updateRegistrationEvent(sla);
}
} catch (ServiceException e) {
throw new CommandException(ErrorCode.E1007, " id " + jobId, e.getMessage(), e);
}
log.debug(
"Job [{0}] reg for SLA. Size of Sla Xml = [{1}]",
jobId, XmlUtils.prettyPrint(eSla).toString().length());
return sla;
}
public void testDaysInMonth() throws Exception {
init("coord-action-create");
String expr = "${coord:daysInMonth(1)}";
assertEquals("30", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:daysInMonth(coord:daysInMonth(1))}";
assertEquals("31", CoordELFunctions.evalAndWrap(eval, expr));
SyncCoordAction appInst = new SyncCoordAction();
SyncCoordDataset ds = new SyncCoordDataset();
;
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.MONTH);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-02T00:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setName("test1");
ds.setType("SYNC");
appInst.setActualTime(DateUtils.parseDateOozieTZ("2009-09-10T23:59Z"));
appInst.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T00:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-02-01T11:00Z"));
appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:daysInMonth(0)}";
assertEquals("28", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:daysInMonth(-1)}";
assertEquals("31", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:daysInMonth(2)}";
assertEquals("30", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:daysInMonth(-3)}";
assertEquals("30", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:daysInMonth(3)}";
assertEquals("31", CoordELFunctions.evalAndWrap(eval, expr));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-02-01T11:00Z")); // Feb
appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:daysInMonth(0)}";
assertEquals("28", CoordELFunctions.evalAndWrap(eval, expr)); // Jan
// 31
// End of Month
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.MONTH);
ds.setEndOfDuration(TimeUnit.END_OF_MONTH);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-02T00:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setName("test1");
ds.setType("SYNC");
appInst.setActualTime(DateUtils.parseDateOozieTZ("2009-09-10T23:59Z"));
appInst.setTimeZone(DateUtils.getTimeZone("UTC"));
// Case 1
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T00:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-02-01T11:00Z"));
appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:daysInMonth(0)}";
assertEquals("28", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:daysInMonth(-1)}";
assertEquals("31", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:daysInMonth(2)}";
assertEquals("30", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:daysInMonth(-3)}";
assertEquals("30", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:daysInMonth(3)}";
assertEquals("31", CoordELFunctions.evalAndWrap(eval, expr));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-02-01T11:00Z")); // Feb
appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:daysInMonth(0)}";
assertEquals("28", CoordELFunctions.evalAndWrap(eval, expr)); // Jan
// 31
}
public void testHoursInDay() throws Exception {
init("coord-action-create");
String expr = "${coord:hoursInDay(1)}";
String res = CoordELFunctions.evalAndWrap(eval, expr);
assertEquals("24", res);
expr = "${coord:hoursInDay(coord:hoursInDay(1))}";
res = CoordELFunctions.evalAndWrap(eval, expr);
SyncCoordAction appInst = new SyncCoordAction();
SyncCoordDataset ds = new SyncCoordDataset();
;
ds.setFrequency(1);
ds.setTimeUnit(TimeUnit.DAY);
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-02T00:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setName("test1");
ds.setType("SYNC");
appInst.setActualTime(DateUtils.parseDateOozieTZ("2009-09-10T23:59Z"));
appInst.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setInitInstance(DateUtils.parseDateOozieTZ("2009-01-01T08:00Z"));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-01-01T08:00Z"));
appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:hoursInDay(0)}";
assertEquals("24", CoordELFunctions.evalAndWrap(eval, expr));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:hoursInDay(0)}";
assertEquals("24", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:hoursInDay(-2)}";
assertEquals("24", CoordELFunctions.evalAndWrap(eval, expr));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-08T08:00Z"));
appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:hoursInDay(0)}";
assertEquals("24", CoordELFunctions.evalAndWrap(eval, expr));
ds.setTimeZone(DateUtils.getTimeZone("Europe/London"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:hoursInDay(0)}";
assertEquals("24", CoordELFunctions.evalAndWrap(eval, expr));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-08T08:00Z"));
appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:hoursInDay(0)}";
assertEquals("23", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:hoursInDay(1)}";
assertEquals("24", CoordELFunctions.evalAndWrap(eval, expr));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-11-01T08:00Z"));
appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:hoursInDay(0)}";
assertEquals("25", CoordELFunctions.evalAndWrap(eval, expr));
appInst.setNominalTime(DateUtils.parseDateOozieTZ("2009-03-08T08:00Z"));
appInst.setActualTime(DateUtils.parseDateOozieTZ("2010-10-01T00:00Z"));
ds.setTimeZone(DateUtils.getTimeZone("America/Los_Angeles"));
ds.setEndOfDuration(TimeUnit.END_OF_DAY);
CoordELFunctions.configureEvaluator(eval, ds, appInst);
expr = "${coord:hoursInDay(0)}";
assertEquals("23", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:hoursInDay(1)}";
assertEquals("24", CoordELFunctions.evalAndWrap(eval, expr));
expr = "${coord:hoursInDay(-1)}";
assertEquals("24", CoordELFunctions.evalAndWrap(eval, expr));
}
/**
* @param offset
* @return n-th available latest instance Date-Time for SYNC data-set
* @throws Exception
*/
private static String coord_latest_sync(int offset) throws Exception {
if (offset > 0) {
throw new RuntimeException(
"For latest there is no meaning " + "of positive instance. n should be <=0" + offset);
}
ELEvaluator eval = ELEvaluator.getCurrent();
String retVal = "";
int datasetFrequency = (int) getDSFrequency(); // in minutes
TimeUnit dsTimeUnit = getDSTimeUnit();
int[] instCount = new int[1];
Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
if (nominalInstanceCal != null) {
Calendar initInstance = getInitialInstanceCal();
SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
if (ds == null) {
throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
}
String uriTemplate = ds.getUriTemplate();
Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
if (conf == null) {
throw new RuntimeException(
"Associated Configuration should be defined with key " + CONFIGURATION);
}
int available = 0;
boolean resolved = false;
String user =
ParamChecker.notEmpty(
(String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
String group =
ParamChecker.notEmpty(
(String) eval.getVariable(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
String doneFlag = ds.getDoneFlag();
while (nominalInstanceCal.compareTo(initInstance) >= 0) {
ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
String uriPath = uriEval.evaluate(uriTemplate, String.class);
String pathWithDoneFlag = uriPath;
if (doneFlag.length() > 0) {
pathWithDoneFlag += "/" + doneFlag;
}
if (isPathAvailable(pathWithDoneFlag, user, group, conf)) {
XLog.getLog(CoordELFunctions.class)
.debug("Found latest(" + available + "): " + pathWithDoneFlag);
if (available == offset) {
XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag);
resolved = true;
retVal = DateUtils.formatDateUTC(nominalInstanceCal);
eval.setVariable("resolved_path", uriPath);
break;
}
available--;
}
// nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(),
// -datasetFrequency);
nominalInstanceCal = (Calendar) initInstance.clone();
instCount[0]--;
nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
// DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
}
if (!resolved) {
// return unchanged latest function with variable 'is_resolved'
// to 'false'
eval.setVariable("is_resolved", Boolean.FALSE);
retVal = "${coord:latest(" + offset + ")}";
} else {
eval.setVariable("is_resolved", Boolean.TRUE);
}
} else { // No feasible nominal time
eval.setVariable("is_resolved", Boolean.FALSE);
}
return retVal;
}
