@Test public void execute_blocking_FAIL() throws KettleException { final long waitTime = 0; OozieJobExecutorConfig config = new OozieJobExecutorConfig(); config.setOozieUrl( "http://localhost:11000/oozie"); // don't worry if it isn't running, we fake out our test // connection to it anyway config.setOozieWorkflowConfig("test-src/job.properties"); config.setJobEntryName("name"); config.setBlockingPollingInterval("1000"); TestOozieJobExecutorJobEntry je = new TestOozieJobExecutorJobEntry(getFailingTest()); je.setParentJob(new Job("test", null, null)); je.setJobConfig(config); Result result = new Result(); long start = System.currentTimeMillis(); je.execute(result, 0); long end = System.currentTimeMillis(); assertTrue( "Total runtime should be >= the wait time if we are blocking", (end - start) >= waitTime); Assert.assertEquals(1, result.getNrErrors()); assertFalse(result.getResult()); }
@Test public void testNrErrorsSuccess() throws Exception { entry.limit = "5"; entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL; entry.tablename = "table"; Result res = entry.execute(new Result(), 0); assertTrue("Eval number of rows should be suceeded", res.getResult()); assertEquals("Apparently there should no error", res.getNrErrors(), 0); // that should work regardless of old/new behavior flag entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y"); res = entry.execute(new Result(), 0); assertTrue("Eval number of rows should be suceeded", res.getResult()); assertEquals("Apparently there should no error", res.getNrErrors(), 0); }
@Test public void testExecuteWithException() throws Exception { entry.arguments = new String[] {null}; Result res = entry.execute(new Result(), 0); assertFalse("Entry should fail", res.getResult()); assertEquals( "File with wrong name was specified. One error should be reported", 1, res.getNrErrors()); }
@Test public void testNrErrorsNoCustomSql() throws Exception { entry.limit = "5"; entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL; entry.iscustomSQL = true; entry.customSQL = null; Result res = entry.execute(new Result(), 0); assertFalse("Eval number of rows should fail", res.getResult()); assertEquals("Apparently there should be an error", res.getNrErrors(), 1); // that should work regardless of old/new behavior flag entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y"); res = entry.execute(new Result(), 0); assertFalse("Eval number of rows should fail", res.getResult()); assertEquals("Apparently there should be an error", res.getNrErrors(), 1); }
@Test public void testSetNrErrorsNewBehaviorFalseResult() throws Exception { // this tests fix for PDI-10270 entry.arguments = new String[] {"nonExistingFile.ext"}; Result res = entry.execute(new Result(), 0); assertFalse("Entry should fail", res.getResult()); assertEquals( "Files not found. Result is false. But... No of errors should be zero", 0, res.getNrErrors()); }
@Test public void testNrErrorsFailureNewBehavior() throws Exception { entry.limit = "1"; entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL; entry.tablename = "table"; Result res = entry.execute(new Result(), 0); assertFalse("Eval number of rows should fail", res.getResult()); assertEquals( "No errors should be reported in result object accoding to the new behavior", res.getNrErrors(), 0); }
@Test public void testSetNrErrorsOldBehaviorFalseResult() throws Exception { // this tests backward compatibility settings for PDI-10270 entry.arguments = new String[] {"nonExistingFile1.ext", "nonExistingFile2.ext"}; entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y"); Result res = entry.execute(new Result(), 0); assertFalse("Entry should fail", res.getResult()); assertEquals( "Files not found. Result is false. And... Number of errors should be the same as number of not found files", entry.arguments.length, res.getNrErrors()); }
@Test public void testNrErrorsFailureOldBehavior() throws Exception { entry.limit = "1"; entry.successCondition = JobEntryEvalTableContent.SUCCESS_CONDITION_ROWS_COUNT_EQUAL; entry.tablename = "table"; entry.setVariable(Const.KETTLE_COMPATIBILITY_SET_ERROR_ON_SPECIFIC_JOB_ENTRIES, "Y"); Result res = entry.execute(new Result(), 0); assertFalse("Eval number of rows should fail", res.getResult()); assertEquals( "An error should be reported in result object accoding to the old behavior", res.getNrErrors(), 1); }
private void executeShell(Result result, List<RowMetaAndData> cmdRows, String[] args) { FileObject fileObject = null; String realScript = null; FileObject tempFile = null; try { // What's the exact command? String[] base = null; List<String> cmds = new ArrayList<String>(); if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobShell.RunningOn", Const.getOS())); } if (insertScript) { realScript = environmentSubstitute(script); } else { String realFilename = environmentSubstitute(getFilename()); fileObject = KettleVFS.getFileObject(realFilename, this); } if (Const.getOS().equals("Windows 95")) { base = new String[] {"command.com", "/C"}; if (insertScript) { tempFile = KettleVFS.createTempFile( "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this); fileObject = createTemporaryShellFile(tempFile, realScript); } } else if (Const.getOS().startsWith("Windows")) { base = new String[] {"cmd.exe", "/C"}; if (insertScript) { tempFile = KettleVFS.createTempFile( "kettle", "shell.bat", System.getProperty("java.io.tmpdir"), this); fileObject = createTemporaryShellFile(tempFile, realScript); } } else { if (insertScript) { tempFile = KettleVFS.createTempFile( "kettle", "shell", System.getProperty("java.io.tmpdir"), this); fileObject = createTemporaryShellFile(tempFile, realScript); } base = new String[] {KettleVFS.getFilename(fileObject)}; } // Construct the arguments... if (argFromPrevious && cmdRows != null) { // Add the base command... for (int i = 0; i < base.length; i++) { cmds.add(base[i]); } if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) { // for windows all arguments including the command itself // need to be // included in 1 argument to cmd/command. StringBuffer cmdline = new StringBuffer(300); cmdline.append('"'); cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject))); // Add the arguments from previous results... for (int i = 0; i < cmdRows.size(); i++) { // Normally just one row, but once in a while to remain compatible we have multiple. RowMetaAndData r = cmdRows.get(i); for (int j = 0; j < r.size(); j++) { cmdline.append(' '); cmdline.append(Const.optionallyQuoteStringByOS(r.getString(j, null))); } } cmdline.append('"'); cmds.add(cmdline.toString()); } else { // Add the arguments from previous results... for (int i = 0; i < cmdRows.size(); i++) { // Normally just one row, but once in a while to remain compatible we have multiple. RowMetaAndData r = cmdRows.get(i); for (int j = 0; j < r.size(); j++) { cmds.add(Const.optionallyQuoteStringByOS(r.getString(j, null))); } } } } else if (args != null) { // Add the base command... for (int i = 0; i < base.length; i++) { cmds.add(base[i]); } if (Const.getOS().equals("Windows 95") || Const.getOS().startsWith("Windows")) { // for windows all arguments including the command itself // need to be // included in 1 argument to cmd/command. StringBuffer cmdline = new StringBuffer(300); cmdline.append('"'); cmdline.append(Const.optionallyQuoteStringByOS(KettleVFS.getFilename(fileObject))); for (int i = 0; i < args.length; i++) { cmdline.append(' '); cmdline.append(Const.optionallyQuoteStringByOS(args[i])); } cmdline.append('"'); cmds.add(cmdline.toString()); } else { for (int i = 0; i < args.length; i++) { cmds.add(args[i]); } } } StringBuffer command = new StringBuffer(); Iterator<String> it = cmds.iterator(); boolean first = true; while (it.hasNext()) { if (!first) { command.append(' '); } else { first = false; } command.append(it.next()); } if (log.isBasic()) { logBasic(BaseMessages.getString(PKG, "JobShell.ExecCommand", command.toString())); } // Build the environment variable list... ProcessBuilder procBuilder = new ProcessBuilder(cmds); Map<String, String> env = procBuilder.environment(); String[] variables = listVariables(); for (int i = 0; i < variables.length; i++) { env.put(variables[i], getVariable(variables[i])); } if (getWorkDirectory() != null && !Const.isEmpty(Const.rtrim(getWorkDirectory()))) { String vfsFilename = environmentSubstitute(getWorkDirectory()); File file = new File(KettleVFS.getFilename(KettleVFS.getFileObject(vfsFilename, this))); procBuilder.directory(file); } Process proc = procBuilder.start(); // any error message? StreamLogger errorLogger = new StreamLogger(log, proc.getErrorStream(), "(stderr)", true); // any output? StreamLogger outputLogger = new StreamLogger(log, proc.getInputStream(), "(stdout)"); // kick them off Thread errorLoggerThread = new Thread(errorLogger); errorLoggerThread.start(); Thread outputLoggerThread = new Thread(outputLogger); outputLoggerThread.start(); proc.waitFor(); if (log.isDetailed()) { logDetailed(BaseMessages.getString(PKG, "JobShell.CommandFinished", command.toString())); } // What's the exit status? result.setExitStatus(proc.exitValue()); if (result.getExitStatus() != 0) { if (log.isDetailed()) { logDetailed( BaseMessages.getString( PKG, "JobShell.ExitStatus", environmentSubstitute(getFilename()), "" + result.getExitStatus())); } result.setNrErrors(1); } // wait until loggers read all data from stdout and stderr errorLoggerThread.join(); outputLoggerThread.join(); // close the streams // otherwise you get "Too many open files, java.io.IOException" after a lot of iterations proc.getErrorStream().close(); proc.getOutputStream().close(); } catch (IOException ioe) { logError( BaseMessages.getString( PKG, "JobShell.ErrorRunningShell", environmentSubstitute(getFilename()), ioe.toString()), ioe); result.setNrErrors(1); } catch (InterruptedException ie) { logError( BaseMessages.getString( PKG, "JobShell.Shellinterupted", environmentSubstitute(getFilename()), ie.toString()), ie); result.setNrErrors(1); } catch (Exception e) { logError( BaseMessages.getString( PKG, "JobShell.UnexpectedError", environmentSubstitute(getFilename()), e.toString()), e); result.setNrErrors(1); } finally { // If we created a temporary file, remove it... // if (tempFile != null) { try { tempFile.delete(); } catch (Exception e) { BaseMessages.getString( PKG, "JobShell.UnexpectedError", tempFile.toString(), e.toString()); } } } if (result.getNrErrors() > 0) { result.setResult(false); } else { result.setResult(true); } }
public Result execute(Result result, int nr) throws KettleException { FileLoggingEventListener loggingEventListener = null; LogLevel shellLogLevel = parentJob.getLogLevel(); if (setLogfile) { String realLogFilename = environmentSubstitute(getLogFilename()); // We need to check here the log filename // if we do not have one, we must fail if (Const.isEmpty(realLogFilename)) { logError(BaseMessages.getString(PKG, "JobEntryShell.Exception.LogFilenameMissing")); result.setNrErrors(1); result.setResult(false); return result; } try { loggingEventListener = new FileLoggingEventListener(getLogChannelId(), realLogFilename, setAppendLogfile); KettleLogStore.getAppender().addLoggingEventListener(loggingEventListener); } catch (KettleException e) { logError( BaseMessages.getString( PKG, "JobEntryShell.Error.UnableopenAppenderFile", getLogFilename(), e.toString())); logError(Const.getStackTracker(e)); result.setNrErrors(1); result.setResult(false); return result; } shellLogLevel = logFileLevel; } log.setLogLevel(shellLogLevel); result.setEntryNr(nr); // "Translate" the arguments for later String[] substArgs = null; if (arguments != null) { substArgs = new String[arguments.length]; for (int idx = 0; idx < arguments.length; idx++) { substArgs[idx] = environmentSubstitute(arguments[idx]); } } int iteration = 0; String[] args = substArgs; RowMetaAndData resultRow = null; boolean first = true; List<RowMetaAndData> rows = result.getRows(); if (log.isDetailed()) { logDetailed( BaseMessages.getString( PKG, "JobEntryShell.Log.FoundPreviousRows", "" + (rows != null ? rows.size() : 0))); } while ((first && !execPerRow) || (execPerRow && rows != null && iteration < rows.size() && result.getNrErrors() == 0)) { first = false; if (rows != null && execPerRow) { resultRow = rows.get(iteration); } else { resultRow = null; } List<RowMetaAndData> cmdRows = null; if (execPerRow) { // Execute for each input row if (argFromPrevious) { // Copy the input row to the (command line) arguments if (resultRow != null) { args = new String[resultRow.size()]; for (int i = 0; i < resultRow.size(); i++) { args[i] = resultRow.getString(i, null); } } } else { // Just pass a single row List<RowMetaAndData> newList = new ArrayList<RowMetaAndData>(); newList.add(resultRow); cmdRows = newList; } } else { if (argFromPrevious) { // Only put the first Row on the arguments args = null; if (resultRow != null) { args = new String[resultRow.size()]; for (int i = 0; i < resultRow.size(); i++) { args[i] = resultRow.getString(i, null); } } else { cmdRows = rows; } } else { // Keep it as it was... cmdRows = rows; } } executeShell(result, cmdRows, args); iteration++; } if (setLogfile) { if (loggingEventListener != null) { KettleLogStore.getAppender().removeLoggingEventListener(loggingEventListener); loggingEventListener.close(); ResultFile resultFile = new ResultFile( ResultFile.FILE_TYPE_LOG, loggingEventListener.getFile(), parentJob.getJobname(), getName()); result.getResultFiles().put(resultFile.getFile().toString(), resultFile); } } return result; }