@Test
public void execute_blocking_FAIL() throws KettleException {
final long waitTime = 0;
OozieJobExecutorConfig config = new OozieJobExecutorConfig();
config.setOozieUrl(
"http://localhost:11000/oozie"); // don't worry if it isn't running, we fake out our test
// connection to it anyway
config.setOozieWorkflowConfig("test-src/job.properties");
config.setJobEntryName("name");
config.setBlockingPollingInterval("1000");
TestOozieJobExecutorJobEntry je = new TestOozieJobExecutorJobEntry(getFailingTest());
je.setParentJob(new Job("test", null, null));
je.setJobConfig(config);
Result result = new Result();
long start = System.currentTimeMillis();
je.execute(result, 0);
long end = System.currentTimeMillis();
assertTrue(
"Total runtime should be >= the wait time if we are blocking", (end - start) >= waitTime);
Assert.assertEquals(1, result.getNrErrors());
assertFalse(result.getResult());
}
@Test
public void testNrErrorsSuccess() throws Exception {
entry.limit = "5";
entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
entry.tablename = "table";
Result res = entry.execute(new Result(), 0);
assertTrue("Eval number of rows should be suceeded", res.getResult());
assertEquals("Apparently there should no error", res.getNrErrors(), 0);
// that should work regardless of old/new behavior flag
entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");
res = entry.execute(new Result(), 0);
assertTrue("Eval number of rows should be suceeded", res.getResult());
assertEquals("Apparently there should no error", res.getNrErrors(), 0);
}
@Test
public void testExecuteWithException() throws Exception {
entry.arguments = new String[] {null};
Result res = entry.execute(new Result(), 0);
assertFalse("Entry should fail", res.getResult());
assertEquals(
"File with wrong name was specified. One error should be reported", 1, res.getNrErrors());
}
@Test
public void testNrErrorsNoCustomSql() throws Exception {
entry.limit = "5";
entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
entry.iscustomSQL = true;
entry.customSQL = null;
Result res = entry.execute(new Result(), 0);
assertFalse("Eval number of rows should fail", res.getResult());
assertEquals("Apparently there should be an error", res.getNrErrors(), 1);
// that should work regardless of old/new behavior flag
entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");
res = entry.execute(new Result(), 0);
assertFalse("Eval number of rows should fail", res.getResult());
assertEquals("Apparently there should be an error", res.getNrErrors(), 1);
}
@Test
public void testSetNrErrorsNewBehaviorFalseResult() throws Exception {
// this tests fix for PDI-10270
entry.arguments = new String[] {"nonExistingFile.ext"};
Result res = entry.execute(new Result(), 0);
assertFalse("Entry should fail", res.getResult());
assertEquals(
"Files not found. Result is false. But... No of errors should be zero",
0,
res.getNrErrors());
}
@Test
public void testNrErrorsFailureNewBehavior() throws Exception {
entry.limit = "1";
entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
entry.tablename = "table";
Result res = entry.execute(new Result(), 0);
assertFalse("Eval number of rows should fail", res.getResult());
assertEquals(
"No errors should be reported in result object accoding to the new behavior",
res.getNrErrors(),
0);
}
@Test
public void testSetNrErrorsOldBehaviorFalseResult() throws Exception {
// this tests backward compatibility settings for PDI-10270
entry.arguments = new String[] {"nonExistingFile1.ext", "nonExistingFile2.ext"};
entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");
Result res = entry.execute(new Result(), 0);
assertFalse("Entry should fail", res.getResult());
assertEquals(
"Files not found. Result is false. And... Number of errors should be the same as number of not found files",
entry.arguments.length,
res.getNrErrors());
}
@Test
public void testNrErrorsFailureOldBehavior() throws Exception {
entry.limit = "1";
entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL;
entry.tablename = "table";
entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y");
Result res = entry.execute(new Result(), 0);
assertFalse("Eval number of rows should fail", res.getResult());
assertEquals(
"An error should be reported in result object accoding to the old behavior",
res.getNrErrors(),
1);
}
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) {
FileObject fileObject = null;
String realScript = null;
FileObject tempFile = null;
try {
// What's the exact command?
String[] base = null;
List<String> cmds = new ArrayList<String>();
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS()));
}
if (insertScript) {
realScript = environmentSubstitute(script);
} else {
String realFilename = environmentSubstitute(getFilename());
fileObject = KettleVFS.getFileObject(realFilename, this);
}
if (Const.getOS().equals("Windows 95")) {
base = new String[] {"command.com", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else if (Const.getOS().startsWith("Windows")) {
base = new String[] {"cmd.exe", "/C"};
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
} else {
if (insertScript) {
tempFile =
KettleVFS.createTempFile(
"kettle", "shell", System.getProperty("java.io.tmpdir"), this);
fileObject = createTemporaryShellFile(tempFile, realScript);
}
base = new String[] {KettleVFS.getFilename(fileObject)};
}
// Construct the arguments...
if (argFromPrevious && cmdRows != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
// Add the arguments from previous results...
for (int i = 0; i < cmdRows.size(); i++) {
// Normally just one row, but once in a while to remain compatible we have multiple.
RowMetaAndData r = cmdRows.get(i);
for (int j = 0; j < r.size(); j++) {
cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null)));
}
}
}
} else if (args != null) {
// Add the base command...
for (int i = 0; i < base.length; i++) {
cmds.add(base[i]);
}
if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) {
// for windows all arguments including the command itself
// need to be
// included in 1 argument to cmd/command.
StringBuffer cmdline = new StringBuffer(300);
cmdline.append('"');
cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject)));
for (int i = 0; i < args.length; i++) {
cmdline.append(' ');
cmdline.append(Const.optionallyQuoteStringByOS(args[i]));
}
cmdline.append('"');
cmds.add(cmdline.toString());
} else {
for (int i = 0; i < args.length; i++) {
cmds.add(args[i]);
}
}
}
StringBuffer command = new StringBuffer();
Iterator<String> it = cmds.iterator();
boolean first = true;
while (it.hasNext()) {
if (!first) {
command.append(' ');
} else {
first = false;
}
command.append(it.next());
}
if (log.isBasic()) {
logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString()));
}
// Build the environment variable list...
ProcessBuilder procBuilder = new ProcessBuilder(cmds);
Map<String, String> env = procBuilder.environment();
String[] variables = listVariables();
for (int i = 0; i < variables.length; i++) {
env.put(variables[i], getVariable(variables[i]));
}
if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) {
String vfsFilename = environmentSubstitute(getWorkDirectory());
File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this)));
procBuilder.directory(file);
}
Process proc = procBuilder.start();
// any error message?
StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true);
// any output?
StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)");
// kick them off
Thread errorLoggerThread = new Thread(errorLogger);
errorLoggerThread.start();
Thread outputLoggerThread = new Thread(outputLogger);
outputLoggerThread.start();
proc.waitFor();
if (log.isDetailed()) {
logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString()));
}
// What's the exit status?
result.setExitStatus(proc.exitValue());
if (result.getExitStatus() != 0) {
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG,
"JobShell.ExitStatus",
environmentSubstitute(getFilename()),
"" + result.getExitStatus()));
}
result.setNrErrors(1);
}
// wait until loggers read all data from stdout and stderr
errorLoggerThread.join();
outputLoggerThread.join();
// close the streams
// otherwise you get "Too many open files, java.io.IOException" after a lot of iterations
proc.getErrorStream().close();
proc.getOutputStream().close();
} catch (IOException ioe) {
logError(
BaseMessages.getString(
PKG,
"JobShell.ErrorRunningShell",
environmentSubstitute(getFilename()),
ioe.toString()),
ioe);
result.setNrErrors(1);
} catch (InterruptedException ie) {
logError(
BaseMessages.getString(
PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()),
ie);
result.setNrErrors(1);
} catch (Exception e) {
logError(
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()),
e);
result.setNrErrors(1);
} finally {
// If we created a temporary file, remove it...
//
if (tempFile != null) {
try {
tempFile.delete();
} catch (Exception e) {
BaseMessages.getString(
PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString());
}
}
}
if (result.getNrErrors() > 0) {
result.setResult(false);
} else {
result.setResult(true);
}
}
public Result execute(Result result, int nr) throws KettleException {
FileLoggingEventListener loggingEventListener = null;
LogLevel shellLogLevel = parentJob.getLogLevel();
if (setLogfile) {
String realLogFilename = environmentSubstitute(getLogFilename());
// We need to check here the log filename
// if we do not have one, we must fail
if (Const.isEmpty(realLogFilename)) {
logError(BaseMessages.getString(PKG, "JobEntryShell.Exception.LogFilenameMissing"));
result.setNrErrors(1);
result.setResult(false);
return result;
}
try {
loggingEventListener =
new FileLoggingEventListener(getLogChannelId(), realLogFilename, setAppendLogfile);
KettleLogStore.getAppender().addLoggingEventListener(loggingEventListener);
} catch (KettleException e) {
logError(
BaseMessages.getString(
PKG, "JobEntryShell.Error.UnableopenAppenderFile", getLogFilename(), e.toString()));
logError(Const.getStackTracker(e));
result.setNrErrors(1);
result.setResult(false);
return result;
}
shellLogLevel = logFileLevel;
}
log.setLogLevel(shellLogLevel);
result.setEntryNr(nr);
// "Translate" the arguments for later
String[] substArgs = null;
if (arguments != null) {
substArgs = new String[arguments.length];
for (int idx = 0; idx < arguments.length; idx++) {
substArgs[idx] = environmentSubstitute(arguments[idx]);
}
}
int iteration = 0;
String[] args = substArgs;
RowMetaAndData resultRow = null;
boolean first = true;
List<RowMetaAndData> rows = result.getRows();
if (log.isDetailed()) {
logDetailed(
BaseMessages.getString(
PKG, "JobEntryShell.Log.FoundPreviousRows", "" + (rows != null ? rows.size() : 0)));
}
while ((first && !execPerRow)
|| (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) {
first = false;
if (rows != null && execPerRow) {
resultRow = rows.get(iteration);
} else {
resultRow = null;
}
List<RowMetaAndData> cmdRows = null;
if (execPerRow) {
// Execute for each input row
if (argFromPrevious) {
// Copy the input row to the (command line) arguments
if (resultRow != null) {
args = new String[resultRow.size()];
for (int i = 0; i < resultRow.size(); i++) {
args[i] = resultRow.getString(i, null);
}
}
} else {
// Just pass a single row
List<RowMetaAndData> newList = new ArrayList<RowMetaAndData>();
newList.add(resultRow);
cmdRows = newList;
}
} else {
if (argFromPrevious) {
// Only put the first Row on the arguments
args = null;
if (resultRow != null) {
args = new String[resultRow.size()];
for (int i = 0; i < resultRow.size(); i++) {
args[i] = resultRow.getString(i, null);
}
} else {
cmdRows = rows;
}
} else {
// Keep it as it was...
cmdRows = rows;
}
}
executeShell(result, cmdRows, args);
iteration++;
}
if (setLogfile) {
if (loggingEventListener != null) {
KettleLogStore.getAppender().removeLoggingEventListener(loggingEventListener);
loggingEventListener.close();
ResultFile resultFile =
new ResultFile(
ResultFile.FILE_TYPE_LOG,
loggingEventListener.getFile(),
parentJob.getJobname(),
getName());
result.getResultFiles().put(resultFile.getFile().toString(), resultFile);
}
}
return result;
}